Skip to main content

rustfs_ecstore/store/
object.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::bucket::lifecycle::{
17    bucket_lifecycle_ops::{enqueue_committed_free_versions, eval_action_from_lifecycle},
18    get_expiry_configs,
19    tier_delete_journal::{
20        ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
21        TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
22        inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
23        record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
24        tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
25    },
26    tier_sweeper::{
27        Jentry, TierDeleteSourceIdentity, attach_tier_delete_source, transitioned_delete_journal_entry_for_source,
28        transitioned_force_delete_journal_entry,
29    },
30};
31use crate::bucket::metadata_sys::{
32    acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
33    get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
34};
35use crate::bucket::object_lock::objectlock_sys::{
36    check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
37};
38use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
39use crate::bucket::versioning::VersioningApi;
40use crate::core::pools::{DecommissionCapacityAdmission, DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
41use crate::diagnostics::object_lock::ObjectLockAttempt;
42use crate::disk::OldCurrentSize;
43use crate::object_api::{
44    NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
45    TierFreeVersionReceiptSink,
46};
47use crate::services::notification_sys::{
48    TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
49    tier_delete_journal_topology_generation,
50};
51use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
52use crate::set_disk::{
53    SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
54    is_lock_optimization_enabled, is_object_lock_diag_enabled,
55};
56use crate::storage_api_contracts::{
57    list::ListOperations as _,
58    namespace::NamespaceLocking as _,
59    object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
60};
61use futures::StreamExt as _;
62use parking_lot::Mutex as ParkingMutex;
63use rustfs_filemeta::ObjectPartInfo;
64use rustfs_io_metrics::{
65    record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
66    record_object_lock_diag_slow_hold,
67};
68use std::{
69    fmt,
70    pin::Pin,
71    sync::atomic::{AtomicBool, Ordering},
72    task::{Context, Poll},
73    time::{Duration, Instant},
74};
75use tokio::io::{AsyncRead, ReadBuf};
76
77#[cfg(not(test))]
78const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
79#[cfg(test)]
80const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
81const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
82const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
83const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
84
85fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
86    if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
87        return None;
88    }
89
90    let sink = TierFreeVersionReceiptSink::new();
91    opts.tier_free_version_receipt_sink = Some(sink.clone());
92    Some(sink)
93}
94
95async fn enqueue_recorded_tier_free_versions(store: &ECStore, sink: Option<TierFreeVersionReceiptSink>) -> usize {
96    let Some(sink) = sink else {
97        return 0;
98    };
99    let Ok(receipts) = sink.drain() else {
100        return 0;
101    };
102    enqueue_committed_free_versions(store, receipts).await
103}
104
105fn build_tier_delete_journal_entry(
106    bucket: &str,
107    object: &str,
108    opts: &ObjectOptions,
109    source: &ObjectInfo,
110) -> Result<Option<Jentry>> {
111    let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
112    let source_object = decode_dir_object(object);
113    let entry = if opts.delete_prefix {
114        transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
115            attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
116            je
117        })
118    } else {
119        transitioned_delete_journal_entry_for_source(
120            version_id,
121            opts.versioned,
122            opts.version_suspended,
123            bucket,
124            source_object.as_str(),
125            source,
126        )
127    };
128    let Some(mut je) = entry else {
129        let lifecycle_exact_object_delete_all = opts.delete_prefix_object && opts.lifecycle_delete_all.is_some();
130        if opts.delete_prefix
131            && !lifecycle_exact_object_delete_all
132            && source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE
133        {
134            return Err(Error::other(
135                "recursive prefix delete cannot preserve a transitioned source with legacy Unknown remote-version state",
136            ));
137        }
138        return Ok(None);
139    };
140    record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
141    if opts.delete_prefix && !je.can_replace_tier_free_version() {
142        return Err(Error::other(
143            "recursive prefix delete requires a stable transitioned source and destination identity",
144        ));
145    }
146    Ok(Some(je))
147}
148
149async fn prepare_prefix_tier_delete_journal_entries(
150    api: &Arc<ECStore>,
151    bucket: &str,
152    prefix: &str,
153    opts: &ObjectOptions,
154) -> Result<PreparedPrefixTierDelete> {
155    Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
156}
157
158type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
159
160fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
161    truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
162}
163
164fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
165    let mut cancelled = false;
166    for result in results {
167        match result {
168            Ok(()) => {}
169            Err(Error::OperationCanceled) => cancelled = true,
170            Err(err) => return Err(err),
171        }
172    }
173    if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
174}
175
176async fn acquire_prefix_tier_delete_reference_leases(
177    api: &Arc<ECStore>,
178    tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
179) -> Result<Vec<TierOperationLease>> {
180    let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
181    tier_references.sort_unstable();
182    let mut leases = Vec::with_capacity(tier_references.len());
183    for (tier_name, backend_identity) in tier_references {
184        let lease = match backend_identity {
185            Some(backend_identity) => {
186                TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
187                    .await
188            }
189            None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
190        }
191        .map_err(Error::other)?;
192        leases.push(lease);
193    }
194    Ok(leases)
195}
196
197async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
198    let tier_references = entries
199        .iter()
200        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
201        .collect::<std::collections::HashSet<_>>();
202    acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
203}
204
205async fn prepare_prefix_tier_delete_journal_entries_inner(
206    api: &Arc<ECStore>,
207    bucket: &str,
208    prefix: &str,
209    opts: &ObjectOptions,
210) -> Result<PreparedPrefixTierDelete> {
211    let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
212        (false, false, None)
213    } else {
214        let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
215        let bucket_fence = opts
216            .bucket_lifecycle_lock_fence
217            .as_ref()
218            .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
219        match Box::pin(inspect_tier_delete_chunk_parent(
220            Arc::clone(api),
221            bucket,
222            bucket_incarnation,
223            prefix,
224            bucket_fence,
225        ))
226        .await?
227        {
228            TierDeleteChunkParentInspection::NoParent => (false, false, None),
229            TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
230            TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
231            TierDeleteChunkParentInspection::Resume(dispatch) => {
232                let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
233                return Ok(PreparedPrefixTierDelete {
234                    dispatch: Some(*dispatch),
235                    chunk_parent_active: true,
236                    chunk_parent_fleet_proof: None,
237                    _leases: leases,
238                });
239            }
240            TierDeleteChunkParentInspection::RetryRequired => {
241                return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
242            }
243        }
244    };
245    let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
246    let mut entries_by_name = std::collections::BTreeMap::new();
247    let batch_limit = tier_delete_dispatch_batch_limit();
248    let logical_prefix = decode_dir_object(prefix);
249    let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
250    let physical_sets = api
251        .pools
252        .iter()
253        .flat_map(|pool| pool.disk_set.iter().cloned())
254        .collect::<Vec<_>>();
255    let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
256    let cancellation = tokio_util::sync::CancellationToken::new();
257    let limit_cancellation = Arc::new(AtomicBool::new(false));
258    let walk_cancel = cancellation.clone();
259    let bucket_owned = bucket.to_string();
260    let prefix_owned = prefix.to_string();
261    let walk = async move {
262        let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
263            let tx = tx.clone();
264            let cancellation = walk_cancel.clone();
265            let bucket = bucket_owned.clone();
266            let prefix = prefix_owned.clone();
267            async move {
268                let result = set
269                    .walk(
270                        cancellation.clone(),
271                        &bucket,
272                        &prefix,
273                        tx,
274                        WalkOptions {
275                            include_free_versions: true,
276                            walkdir_timeout: Some(Duration::ZERO),
277                            ..Default::default()
278                        },
279                    )
280                    .await;
281                if result.is_err() {
282                    cancellation.cancel();
283                }
284                result
285            }
286        }))
287        .buffer_unordered(4)
288        .collect::<Vec<_>>()
289        .await;
290        drop(tx);
291        combine_tier_delete_walk_results(results)
292    };
293    let collect_limit_cancellation = limit_cancellation.clone();
294    let collect = async {
295        let mut truncated = false;
296        while let Some(result) = rx.recv().await {
297            if let Some(err) = result.err {
298                // Once limit + 1 has been observed this request can authorize
299                // only the exact retained batch; it cannot infer prefix
300                // absence or run the raw delete. Drain only the explicit
301                // cancellation fallout; a real walker error must still fail
302                // the request even when another set reached the limit first.
303                if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
304                {
305                    continue;
306                }
307                cancellation.cancel();
308                return Err(err);
309            }
310            if truncated {
311                continue;
312            }
313            let Some(source) = result.item else {
314                continue;
315            };
316            let object = decode_dir_object(&source.name);
317            if exact_object.as_ref().is_some_and(|expected| expected != &object) {
318                continue;
319            }
320            if exact_object.is_none() && !object.starts_with(&logical_prefix) {
321                continue;
322            }
323            if source.transitioned_object.free_version {
324                cancellation.cancel();
325                return Err(Error::other(
326                    "recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
327                ));
328            }
329            let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
330                let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
331                Some((source.transitioned_object.tier.clone(), backend_identity))
332            } else {
333                None
334            };
335            if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
336                let name = tier_delete_journal_object_name(&entry);
337                let at_limit = entries_by_name.len() == batch_limit;
338                match entries_by_name.entry(name) {
339                    std::collections::btree_map::Entry::Occupied(_) => {}
340                    std::collections::btree_map::Entry::Vacant(_) if at_limit => {
341                        truncated = true;
342                        collect_limit_cancellation.store(true, Ordering::Release);
343                        cancellation.cancel();
344                    }
345                    std::collections::btree_map::Entry::Vacant(slot) => {
346                        if let Some(tier_reference) = tier_reference {
347                            tier_references.insert(tier_reference);
348                        }
349                        slot.insert(entry);
350                    }
351                }
352            } else if let Some(tier_reference) = tier_reference {
353                tier_references.insert(tier_reference);
354            }
355        }
356        Ok(truncated)
357    };
358    let (walk_result, collect_result) = tokio::join!(walk, collect);
359    let truncated = collect_result?;
360    // A truncated walk normally reports OperationCanceled from the physical
361    // walkers. That cancellation is expected; any other result is a genuine
362    // scan failure and cannot be hidden by the bounded batch.
363    if let Err(err) = walk_result
364        && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
365    {
366        return Err(err);
367    }
368    let entries = entries_by_name.into_values().collect::<Vec<_>>();
369    let mut leased_tier_references = tier_references;
370    let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
371    if entries.is_empty() {
372        let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
373            let fleet_proof = acquire_tier_delete_journal_fleet_proof()
374                .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
375            if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
376                return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
377            }
378            Some(fleet_proof)
379        } else {
380            None
381        };
382        return Ok(PreparedPrefixTierDelete {
383            dispatch: None,
384            chunk_parent_active,
385            chunk_parent_fleet_proof,
386            _leases: leases,
387        });
388    }
389
390    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
391    let bucket_fence = opts
392        .bucket_lifecycle_lock_fence
393        .as_ref()
394        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
395    let fleet_proof = acquire_tier_delete_journal_fleet_proof()
396        .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
397    if chunk_parent_topology_generation
398        .as_deref()
399        .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
400    {
401        return Err(Error::other("tier delete chunk parent topology changed during source scan"));
402    }
403    let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
404        Box::pin(prepare_tier_delete_chunk_dispatch(
405            Arc::clone(api),
406            bucket,
407            bucket_incarnation,
408            prefix,
409            entries,
410            truncated && !chunk_parent_active,
411            fleet_proof,
412            bucket_fence,
413        ))
414        .await?
415    } else if legacy_manifest_active && truncated {
416        resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
417            .await?
418    } else {
419        prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
420            .await?
421    };
422    // A resumed legacy authorization may own predecessors that are absent
423    // from this bounded scan. Pin every backend generation in the actual
424    // permit before any local mutation, while avoiding duplicate leases for
425    // entries already covered by the scan.
426    let additional_tier_references = dispatch
427        .entries()?
428        .iter()
429        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
430        .filter(|reference| leased_tier_references.insert(reference.clone()))
431        .collect::<std::collections::HashSet<_>>();
432    leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
433    if legacy_manifest_active && truncated {
434        dispatch.require_exact_predecessor_replay();
435    }
436    Ok(PreparedPrefixTierDelete {
437        dispatch: Some(dispatch),
438        chunk_parent_active: chunk_parent_active || truncated,
439        chunk_parent_fleet_proof: None,
440        _leases: leases,
441    })
442}
443
444struct PreparedPrefixTierDelete {
445    dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
446    chunk_parent_active: bool,
447    chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
448    _leases: Vec<TierOperationLease>,
449}
450
451#[cfg(all(test, feature = "test-util"))]
452struct PrefixDeleteAfterJournalPrepareBarrierState {
453    bucket: String,
454    prefix: String,
455    arrived: tokio::sync::Notify,
456    release: tokio::sync::Notify,
457}
458
459#[cfg(all(test, feature = "test-util"))]
460pub(crate) struct PrefixDeleteAfterJournalPrepareBarrier {
461    state: Arc<PrefixDeleteAfterJournalPrepareBarrierState>,
462}
463
464#[cfg(all(test, feature = "test-util"))]
465static PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER: std::sync::OnceLock<
466    std::sync::Mutex<Option<Arc<PrefixDeleteAfterJournalPrepareBarrierState>>>,
467> = std::sync::OnceLock::new();
468
469#[cfg(all(test, feature = "test-util"))]
470impl PrefixDeleteAfterJournalPrepareBarrier {
471    pub(crate) fn install(bucket: &str, prefix: &str) -> Self {
472        let state = Arc::new(PrefixDeleteAfterJournalPrepareBarrierState {
473            bucket: bucket.to_string(),
474            prefix: prefix.to_string(),
475            arrived: tokio::sync::Notify::new(),
476            release: tokio::sync::Notify::new(),
477        });
478        let mut slot = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
479            .get_or_init(|| std::sync::Mutex::new(None))
480            .lock()
481            .expect("prefix journal barrier mutex should not poison");
482        assert!(slot.is_none(), "prefix journal barrier must not already be installed");
483        *slot = Some(Arc::clone(&state));
484        Self { state }
485    }
486
487    pub(crate) async fn wait_until_paused(&self) {
488        self.state.arrived.notified().await;
489    }
490
491    pub(crate) fn release(&self) {
492        self.state.release.notify_one();
493    }
494}
495
496#[cfg(all(test, feature = "test-util"))]
497impl Drop for PrefixDeleteAfterJournalPrepareBarrier {
498    fn drop(&mut self) {
499        self.state.release.notify_one();
500        if let Some(slot) = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER.get() {
501            let mut slot = slot.lock().expect("prefix journal barrier mutex should not poison");
502            if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
503                *slot = None;
504            }
505        }
506    }
507}
508
509#[cfg(all(test, feature = "test-util"))]
510async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) {
511    let state = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
512        .get_or_init(|| std::sync::Mutex::new(None))
513        .lock()
514        .expect("prefix journal barrier mutex should not poison")
515        .as_ref()
516        .filter(|state| state.bucket == bucket && state.prefix == prefix)
517        .cloned();
518    if let Some(state) = state {
519        state.arrived.notify_one();
520        state.release.notified().await;
521    }
522}
523
524async fn delete_prefix_with_tier_delete_journal(
525    store: &ECStore,
526    bucket: &str,
527    object: &str,
528    opts: &ObjectOptions,
529    tier_journal_api: Option<&Arc<ECStore>>,
530) -> Result<()> {
531    let Some(api) = tier_journal_api else {
532        return store.delete_prefix(bucket, object, opts).await;
533    };
534    let PreparedPrefixTierDelete {
535        dispatch,
536        chunk_parent_active,
537        chunk_parent_fleet_proof,
538        _leases,
539    } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
540    let Some(dispatch) = dispatch else {
541        // There is no new remote-cleanup candidate. Keep compatibility-path
542        // tier leases and, for a chunked final pass, the matching parent fleet
543        // proof alive until local deletion and parent completion both commit.
544        let _tier_leases = _leases;
545        let mut operation_opts = opts.clone();
546        // `tier_delete_journal_api` means a v6 dispatch authorization must be
547        // present at every destructive phase.  With no v6 candidate, clear
548        // that marker so ordinary objects use the local path and legacy
549        // transitioned metadata retains its FreeVersion fallback.
550        operation_opts.tier_delete_journal_api = None;
551        operation_opts.tier_delete_dispatch_authorization = None;
552        let parent_fleet_proof = if chunk_parent_active {
553            Some(
554                chunk_parent_fleet_proof
555                    .as_ref()
556                    .filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
557                    .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
558            )
559        } else {
560            None
561        };
562        store.delete_prefix(bucket, object, &operation_opts).await?;
563        if let Some(parent_fleet_proof) = parent_fleet_proof {
564            #[cfg(all(test, feature = "test-util"))]
565            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
566                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
567            )
568            .await;
569            let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
570            let bucket_fence = opts
571                .bucket_lifecycle_lock_fence
572                .as_ref()
573                .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
574            if !Box::pin(complete_tier_delete_chunk_parent(
575                Arc::clone(api),
576                bucket,
577                bucket_incarnation,
578                object,
579                bucket_fence,
580                parent_fleet_proof,
581            ))
582            .await?
583            {
584                return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
585            }
586        }
587        return Ok(());
588    };
589    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
590    let bucket_fence = opts
591        .bucket_lifecycle_lock_fence
592        .as_ref()
593        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
594    let active = dispatch.consume(bucket, bucket_incarnation, object)?;
595    let mut operation_opts = opts.clone();
596    operation_opts.tier_delete_dispatch_authorization = Some(active.authorization());
597
598    #[cfg(all(test, feature = "test-util"))]
599    pause_prefix_delete_after_journal_prepare(bucket, object).await;
600
601    // Keep every backend generation lease until the whole local operation has
602    // either committed its journal set or returned an ambiguous mutation.
603    let _tier_leases = _leases;
604    if active.predecessor_replay_required() || active.is_chunked() {
605        replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
606        #[cfg(all(test, feature = "test-util"))]
607        if active.is_chunked() {
608            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
609                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
610            )
611            .await;
612        }
613        complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
614        if active.is_chunked() {
615            return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
616        }
617        return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
618    }
619    let result = store.delete_prefix(bucket, object, &operation_opts).await;
620    match result {
621        Ok(()) => {
622            if let Err(err) = complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await {
623                warn!(
624                    event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
625                    component = LOG_COMPONENT_ECSTORE,
626                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
627                    bucket,
628                    prefix = object,
629                    error = ?err,
630                    "Prefix deletion committed with durable tier cleanup pending"
631                );
632            }
633            Ok(())
634        }
635        // Once DispatchAuthorized exists, every failure is mutation-ambiguous.
636        // Recovery decides from physical proof; never abort the journals here.
637        Err(err) => Err(err),
638    }
639}
640
641async fn replay_authorized_tier_delete_sources(
642    store: &ECStore,
643    bucket: &str,
644    prefix: &str,
645    active: &ActiveTierDeleteDispatch,
646    opts: &ObjectOptions,
647) -> Result<()> {
648    let _publication_scope_guard = opts
649        .scanner_publication_commit_scope
650        .clone()
651        .map(ScannerPublicationCommitScopeGuard::new);
652    let publication_scope = opts.scanner_publication_commit_scope.as_ref();
653    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
654    let bucket_fence = opts
655        .bucket_lifecycle_lock_fence
656        .as_ref()
657        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
658    let authorization = active.authorization();
659    authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
660
661    let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
662    for entry in active.entries() {
663        let source = entry
664            .source
665            .as_ref()
666            .filter(|source| source.has_stable_identity())
667            .ok_or_else(|| Error::other("authorized tier delete predecessor has no stable source"))?;
668        if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
669            return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
670        }
671        source_objects.insert(source.object.as_str());
672    }
673
674    if let Some(scope) = publication_scope {
675        if scope.state() == ScannerPublicationCommitState::Admitted {
676            scope
677                .try_begin()
678                .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
679        }
680        if !scope.can_commit() {
681            let _ = scope.mark_indeterminate();
682            return Err(StorageError::OperationCanceled);
683        }
684    }
685
686    let stopped = Arc::new(AtomicBool::new(false));
687    // The caller holds bucket lifecycle WRITE. Each bounded future acquires
688    // only one logical object's physical lock set and releases it before
689    // completion; no future nests locks for two object keys.
690    let make_replay = |object: String| {
691        let stopped = stopped.clone();
692        let authorization = authorization.clone();
693        async move {
694            if stopped.load(Ordering::Acquire) {
695                return Ok::<_, Error>(0usize);
696            }
697            let result = async {
698                if bucket_fence.is_lock_lost() {
699                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
700                }
701                let encoded_object = encode_dir_object(&object);
702                let guards = if opts.delete_prefix_object {
703                    store
704                        .acquire_remaining_physical_object_write_locks(
705                            "tier_delete_dispatch_predecessor_replay",
706                            bucket,
707                            &encoded_object,
708                        )
709                        .await?
710                } else {
711                    store
712                        .acquire_all_physical_object_write_locks(
713                            "tier_delete_dispatch_predecessor_replay",
714                            bucket,
715                            &encoded_object,
716                        )
717                        .await?
718                };
719                authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
720                if publication_scope.is_some_and(|scope| !scope.can_commit()) {
721                    return Err(StorageError::OperationCanceled);
722                }
723                let mut replay_opts = opts.clone();
724                replay_opts.no_lock = true;
725                replay_opts.delete_prefix = false;
726                replay_opts.delete_prefix_object = false;
727                for guard in &guards {
728                    guard.add_namespace_lock_fence(&mut replay_opts);
729                }
730                let mut deleted = 0usize;
731                for pool in &store.pools {
732                    for set in &pool.disk_set {
733                        authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
734                        deleted = deleted
735                            .checked_add(
736                                set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
737                                    .await?,
738                            )
739                            .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
740                    }
741                }
742                if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
743                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
744                }
745                Ok(deleted)
746            }
747            .await;
748            if result.is_err() {
749                stopped.store(true, Ordering::Release);
750            }
751            result
752        }
753    };
754    let mut objects = source_objects.into_iter();
755    let mut replays = futures::stream::FuturesUnordered::new();
756    for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
757        let Some(object) = objects.next().map(ToOwned::to_owned) else {
758            break;
759        };
760        replays.push(make_replay(object));
761    }
762    let mut deleted = 0usize;
763    let mut first_error = None;
764    while let Some(result) = replays.next().await {
765        match result {
766            Ok(count) => {
767                deleted = deleted
768                    .checked_add(count)
769                    .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
770            }
771            Err(err) if first_error.is_none() => first_error = Some(err),
772            Err(_) => {}
773        }
774        if first_error.is_none()
775            && !stopped.load(Ordering::Acquire)
776            && let Some(object) = objects.next().map(ToOwned::to_owned)
777        {
778            replays.push(make_replay(object));
779        }
780    }
781    if deleted > 0 {
782        super::list_objects::observe_list_objects_mutation(store, bucket).await;
783    }
784    if let Some(err) = first_error {
785        if publication_scope.is_some_and(|scope| !scope.can_commit())
786            && let Some(scope) = publication_scope
787        {
788            let _ = scope.mark_indeterminate();
789        }
790        return Err(err);
791    }
792    if let Some(scope) = publication_scope {
793        let _ = scope.mark_committed();
794    }
795    Ok(())
796}
797
798fn tier_delete_source_matches_replay_scope(
799    source: &TierDeleteSourceIdentity,
800    bucket: &str,
801    persisted_prefix: &str,
802    exact_object: bool,
803) -> bool {
804    tier_delete_source_matches_dispatch_scope(source, bucket, persisted_prefix)
805        && (!exact_object || source.object == decode_dir_object(persisted_prefix))
806}
807
808async fn delete_recursive_prefix_with_tier_delete_journal(
809    store: &ECStore,
810    bucket: &str,
811    object: &str,
812    opts: &ObjectOptions,
813    tier_journal_api: Option<&Arc<ECStore>>,
814) -> Result<()> {
815    // Prefix deletes cover multiple object keys; an exact lock on the prefix
816    // string would not protect child objects.
817    if !is_meta_bucketname(bucket) {
818        let state = opts
819            .object_lock_config_snapshot
820            .as_deref()
821            .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))?
822            .state();
823        ensure_recursive_force_delete_allowed_for_state(bucket, state)?;
824        let bypass_governance = opts
825            .object_lock_delete
826            .as_ref()
827            .is_some_and(|delete_opts| delete_opts.bypass_governance);
828        for pool in &store.pools {
829            for set in &pool.disk_set {
830                let mut marker = None;
831                let mut version_marker = None;
832                loop {
833                    let page = set
834                        .clone()
835                        .inner_list_object_versions_for_recursive_delete(
836                            bucket,
837                            object,
838                            marker.clone(),
839                            version_marker.clone(),
840                            RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
841                        )
842                        .await?;
843                    for object_info in &page.objects {
844                        if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() {
845                            return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone()));
846                        }
847                    }
848                    if !page.is_truncated {
849                        break;
850                    }
851                    let next_marker = page
852                        .next_marker
853                        .ok_or_else(|| Error::other("recursive delete version scan did not return a continuation marker"))?;
854                    if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker {
855                        return Err(Error::other("recursive delete version scan did not advance"));
856                    }
857                    marker = Some(next_marker);
858                    version_marker = page.next_version_idmarker;
859                }
860            }
861        }
862    }
863    // A trailing slash selects a directory, not the object at its parent key.
864    // Raw filesystem recursion would also remove that object's metadata and
865    // data. Preserve it by purging the selected keys individually when they
866    // share this physical directory. The bucket write lock covers both scans.
867    if object.ends_with('/') && !is_meta_bucketname(bucket) {
868        let parent = object.strip_suffix('/').unwrap_or(object);
869        for pool in &store.pools {
870            for set in &pool.disk_set {
871                let page = set
872                    .clone()
873                    .inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
874                    .await?;
875                if page.objects.iter().any(|info| info.name == parent) {
876                    return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
877                }
878            }
879        }
880    }
881    delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
882}
883
884async fn delete_directory_keys_with_tier_delete_journal(
885    store: &ECStore,
886    bucket: &str,
887    prefix: &str,
888    opts: &ObjectOptions,
889    tier_journal_api: Option<&Arc<ECStore>>,
890) -> Result<()> {
891    for pool in &store.pools {
892        for set in &pool.disk_set {
893            let mut previous_keys = std::collections::BTreeSet::new();
894            loop {
895                // Restart after each bounded batch: its version markers have
896                // been deleted, and the bucket write lock excludes new keys.
897                let page = set
898                    .clone()
899                    .inner_list_object_versions_for_recursive_delete(
900                        bucket,
901                        prefix,
902                        None,
903                        None,
904                        RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
905                    )
906                    .await?;
907                let keys = page
908                    .objects
909                    .into_iter()
910                    .map(|info| info.name)
911                    .filter(|key| key.starts_with(prefix))
912                    .collect::<std::collections::BTreeSet<_>>();
913                if keys.is_empty() {
914                    break;
915                }
916                if keys == previous_keys {
917                    return Err(Error::other("directory deletion did not advance"));
918                }
919                for key in &keys {
920                    let encoded_key = encode_dir_object(key);
921                    let mut exact_opts = opts.clone();
922                    exact_opts.delete_prefix_object = true;
923                    let _guard = store
924                        .acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
925                        .await?;
926                    delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
927                }
928                previous_keys = keys;
929            }
930        }
931    }
932    Ok(())
933}
934
935/// A GET whose object identity has been resolved while its namespace read lock
936/// remains held, but whose body reader has not been constructed yet.
937///
938/// The application can evaluate request preconditions and cache coordination
939/// against [`Self::object_info`] before consuming this value. Dropping it
940/// releases the read lock without constructing a body reader.
941pub struct PreparedGetObjectReader {
942    pool: Arc<Sets>,
943    bucket: String,
944    object: String,
945    range: Option<HTTPRangeSpec>,
946    headers: HeaderMap,
947    opts: ObjectOptions,
948    metadata: crate::set_disk::PreparedGetObjectMetadata,
949    read_lock_guard: Option<ObjectLockDiagGuard>,
950}
951
952impl PreparedGetObjectReader {
953    /// Returns the fresh metadata snapshot protected by this prepared read.
954    pub fn object_info(&self) -> &ObjectInfo {
955        self.metadata.object_info()
956    }
957
958    /// Finishes a metadata-only decision without constructing a body reader.
959    pub fn into_object_info(mut self) -> ObjectInfo {
960        self.metadata.take_object_info()
961    }
962
963    /// Replaces the headers used when this prepared value constructs its body reader.
964    #[must_use]
965    pub fn with_headers(mut self, headers: HeaderMap) -> Self {
966        self.headers = headers;
967        self
968    }
969
970    /// Constructs the body reader while retaining the metadata snapshot's read
971    /// lock, then transfers that lock to the returned stream as usual. The
972    /// staged caller already performed the authoritative app-layer cache probe,
973    /// so the nested set-disk reader must not probe the same hook again.
974    pub async fn into_reader(self) -> Result<GetObjectReader> {
975        let mut reader =
976            crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
977                &self.bucket,
978                &self.object,
979                self.range,
980                self.headers,
981                &self.opts,
982                self.metadata,
983            ))
984            .await?;
985        reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
986        Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
987    }
988}
989
990struct LockGuardedReader {
991    inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
992    guard: Option<ObjectLockDiagGuard>,
993}
994
995impl AsyncRead for LockGuardedReader {
996    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
997        let had_capacity = buf.remaining() > 0;
998        let filled_before = buf.filled().len();
999        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1000        if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
1001            self.guard.take();
1002        }
1003        poll
1004    }
1005}
1006
1007#[derive(Clone, Copy, Debug)]
1008enum ObjectLockDiagMode {
1009    Read,
1010    Write,
1011}
1012
1013impl ObjectLockDiagMode {
1014    fn as_str(self) -> &'static str {
1015        match self {
1016            Self::Read => "read",
1017            Self::Write => "write",
1018        }
1019    }
1020}
1021
1022impl fmt::Display for ObjectLockDiagMode {
1023    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1024        f.write_str(self.as_str())
1025    }
1026}
1027
1028pub(crate) struct ObjectLockDiagGuard {
1029    guard: rustfs_lock::NamespaceLockGuard,
1030    #[cfg(test)]
1031    test_namespace_lock_fence: Option<NamespaceLockFence>,
1032    enabled: bool,
1033    op: &'static str,
1034    bucket: Option<String>,
1035    object: Option<String>,
1036    owner: Option<String>,
1037    mode: ObjectLockDiagMode,
1038    acquired_at: Instant,
1039    _attempt: ObjectLockAttempt,
1040}
1041
1042impl ObjectLockDiagGuard {
1043    fn new(
1044        guard: rustfs_lock::NamespaceLockGuard,
1045        enabled: bool,
1046        op: &'static str,
1047        bucket: Option<String>,
1048        object: Option<String>,
1049        owner: Option<String>,
1050        mode: ObjectLockDiagMode,
1051    ) -> Self {
1052        Self {
1053            guard,
1054            #[cfg(test)]
1055            test_namespace_lock_fence: None,
1056            enabled,
1057            op,
1058            bucket,
1059            object,
1060            owner,
1061            mode,
1062            acquired_at: Instant::now(),
1063            _attempt: ObjectLockAttempt::default(),
1064        }
1065    }
1066
1067    fn with_attempt(mut self, attempt: ObjectLockAttempt) -> Self {
1068        self._attempt = attempt;
1069        self
1070    }
1071
1072    pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
1073        match &self.guard {
1074            rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
1075            rustfs_lock::NamespaceLockGuard::Fast(_) => None,
1076        }
1077    }
1078
1079    pub(crate) fn is_lock_lost(&self) -> bool {
1080        self.guard.is_lock_lost() || {
1081            #[cfg(test)]
1082            {
1083                self.test_namespace_lock_fence
1084                    .as_ref()
1085                    .is_some_and(NamespaceLockFence::is_lock_lost)
1086            }
1087            #[cfg(not(test))]
1088            {
1089                false
1090            }
1091        }
1092    }
1093
1094    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1095        opts.ensure_namespace_lock_fence();
1096        if let Some(signal) = self.lock_lost_signal() {
1097            opts.add_namespace_lock_lost_signal(signal);
1098        }
1099        #[cfg(test)]
1100        if let Some(fence) = self.test_namespace_lock_fence.as_ref() {
1101            opts.add_namespace_lock_fence(fence);
1102        }
1103    }
1104}
1105
1106#[cfg(test)]
1107#[derive(Clone, Copy, PartialEq, Eq)]
1108pub(crate) enum DecommissionMutationFenceTestPhase {
1109    Migration,
1110    SourceCleanup,
1111}
1112
1113#[cfg(test)]
1114struct DecommissionMutationFenceLossState {
1115    bucket: String,
1116    object: String,
1117    phase: DecommissionMutationFenceTestPhase,
1118    fence: NamespaceLockFence,
1119    loss_handle: Arc<std::sync::atomic::AtomicBool>,
1120}
1121
1122#[cfg(test)]
1123pub(crate) struct DecommissionMutationFenceLossHook {
1124    state: Arc<DecommissionMutationFenceLossState>,
1125}
1126
1127#[cfg(test)]
1128static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock<
1129    std::sync::Mutex<Option<Arc<DecommissionMutationFenceLossState>>>,
1130> = std::sync::OnceLock::new();
1131
1132#[cfg(test)]
1133impl DecommissionMutationFenceLossHook {
1134    pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self {
1135        let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
1136        let state = Arc::new(DecommissionMutationFenceLossState {
1137            bucket: bucket.to_string(),
1138            object: object.to_string(),
1139            phase,
1140            fence,
1141            loss_handle,
1142        });
1143        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1144            .get_or_init(|| std::sync::Mutex::new(None))
1145            .lock()
1146            .expect("decommission mutation fence loss hooks should not poison");
1147        assert!(slot.is_none(), "decommission mutation fence loss hook must be unique");
1148        *slot = Some(Arc::clone(&state));
1149        Self { state }
1150    }
1151
1152    pub(crate) fn mark_lost(&self) {
1153        self.state.loss_handle.store(true, Ordering::Release);
1154    }
1155}
1156
1157#[cfg(test)]
1158impl Drop for DecommissionMutationFenceLossHook {
1159    fn drop(&mut self) {
1160        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1161            .get_or_init(|| std::sync::Mutex::new(None))
1162            .lock()
1163            .expect("decommission mutation fence loss hooks should not poison");
1164        if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) {
1165            *slot = None;
1166        }
1167    }
1168}
1169
1170#[cfg(test)]
1171fn decommission_mutation_fence_for_test(
1172    bucket: &str,
1173    object: &str,
1174    phase: DecommissionMutationFenceTestPhase,
1175) -> Option<NamespaceLockFence> {
1176    DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1177        .get_or_init(|| std::sync::Mutex::new(None))
1178        .lock()
1179        .expect("decommission mutation fence loss hooks should not poison")
1180        .as_ref()
1181        .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase)
1182        .map(|hook| hook.fence.clone())
1183}
1184
1185#[cfg(all(test, feature = "test-util"))]
1186struct DecommissionFreeVersionSourceRaceState {
1187    bucket: String,
1188    object: String,
1189    arrived: tokio::sync::Notify,
1190    release: tokio::sync::Notify,
1191}
1192
1193#[cfg(all(test, feature = "test-util"))]
1194pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
1195    state: Arc<DecommissionFreeVersionSourceRaceState>,
1196}
1197
1198#[cfg(all(test, feature = "test-util"))]
1199static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
1200    std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
1201> = std::sync::OnceLock::new();
1202
1203#[cfg(all(test, feature = "test-util"))]
1204impl DecommissionFreeVersionSourceRaceBarrier {
1205    pub(crate) fn install(bucket: &str, object: &str) -> Self {
1206        let state = Arc::new(DecommissionFreeVersionSourceRaceState {
1207            bucket: bucket.to_string(),
1208            object: object.to_string(),
1209            arrived: tokio::sync::Notify::new(),
1210            release: tokio::sync::Notify::new(),
1211        });
1212        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1213            .get_or_init(|| std::sync::Mutex::new(None))
1214            .lock()
1215            .expect("decommission free-version source race barrier should not poison");
1216        assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
1217        *slot = Some(Arc::clone(&state));
1218        Self { state }
1219    }
1220
1221    pub(crate) async fn wait_until_paused(&self) {
1222        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1223            .await
1224            .expect("decommission should pause before acquiring the free-version source lock");
1225    }
1226
1227    pub(crate) fn release(&self) {
1228        self.state.release.notify_one();
1229    }
1230}
1231
1232#[cfg(all(test, feature = "test-util"))]
1233impl Drop for DecommissionFreeVersionSourceRaceBarrier {
1234    fn drop(&mut self) {
1235        self.state.release.notify_one();
1236        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1237            .get_or_init(|| std::sync::Mutex::new(None))
1238            .lock()
1239            .expect("decommission free-version source race barrier should not poison");
1240        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1241            *slot = None;
1242        }
1243    }
1244}
1245
1246#[cfg(all(test, feature = "test-util"))]
1247async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
1248    let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1249        .get_or_init(|| std::sync::Mutex::new(None))
1250        .lock()
1251        .expect("decommission free-version source race barrier should not poison")
1252        .as_ref()
1253        .filter(|state| state.bucket == bucket && state.object == object)
1254        .cloned();
1255    if let Some(state) = state {
1256        state.arrived.notify_one();
1257        state.release.notified().await;
1258    }
1259}
1260
1261pub(crate) struct SourceCleanupMutationFence {
1262    guard: ObjectLockDiagGuard,
1263    source_lock_covered: bool,
1264}
1265
1266impl SourceCleanupMutationFence {
1267    pub(crate) fn source_lock_covered(&self) -> bool {
1268        self.source_lock_covered
1269    }
1270
1271    pub(crate) fn is_lock_lost(&self) -> bool {
1272        self.guard.is_lock_lost()
1273    }
1274
1275    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1276        self.guard.add_namespace_lock_fence(opts);
1277    }
1278}
1279
1280/// Opaque write-lock guard for the RestoreObject accept path; see
1281/// [`ECStore::acquire_restore_accept_guard`]. Deliberately not a general lock
1282/// API — it only exists so the accept path's restore-status compare-and-set
1283/// can span the ecstore/API layer boundary.
1284pub struct RestoreAcceptGuard(ObjectLockDiagGuard);
1285
1286impl RestoreAcceptGuard {
1287    /// True when the underlying namespace lock was lost (e.g. heartbeat
1288    /// refresh lost quorum). Callers must check this before committing the
1289    /// restore-status write the guard exists to serialize.
1290    pub fn is_lock_lost(&self) -> bool {
1291        self.0.guard.is_lock_lost()
1292    }
1293
1294    pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1295        self.0.add_namespace_lock_fence(opts);
1296    }
1297}
1298
1299/// Liveness proof for one asynchronous restore generation. Unlike the object
1300/// accept/commit locks, this lock lives in the reserved metadata namespace and
1301/// never blocks reads or unrelated writes of the restored object.
1302pub struct RestoreWorkerGuard(rustfs_lock::NamespaceLockGuard);
1303
1304impl RestoreWorkerGuard {
1305    pub fn is_lock_lost(&self) -> bool {
1306        self.0.is_lock_lost()
1307    }
1308}
1309
1310impl Drop for ObjectLockDiagGuard {
1311    fn drop(&mut self) {
1312        if !self.enabled || self.guard.is_released() {
1313            return;
1314        }
1315
1316        let hold = self.acquired_at.elapsed();
1317        record_object_lock_diag_hold_duration(self.op, self.mode.as_str(), hold);
1318        let threshold = get_object_lock_diag_slow_hold_threshold();
1319        if hold >= threshold {
1320            record_object_lock_diag_slow_hold(self.op, self.mode.as_str());
1321            warn!(
1322                target: "rustfs_ecstore::object_lock_diag",
1323                op = self.op,
1324                bucket = %self.bucket.as_deref().unwrap_or_default(),
1325                object = %self.object.as_deref().unwrap_or_default(),
1326                mode = %self.mode,
1327                owner = %self.owner.as_deref().unwrap_or_default(),
1328                hold_ms = hold.as_millis(),
1329                threshold_ms = threshold.as_millis(),
1330                "object namespace lock held longer than threshold"
1331            );
1332        }
1333    }
1334}
1335
1336/// A failure to preserve one object generation for a SelectObjectContent read.
1337#[derive(Debug, thiserror::Error)]
1338#[non_exhaustive]
1339pub enum SnapshotConsistencyError {
1340    #[error("namespace locking is disabled for SelectObjectContent")]
1341    LockingDisabled,
1342    #[error("SelectObjectContent namespace lock was lost")]
1343    LockLost,
1344    #[error("object read semantics changed while SelectObjectContent was running")]
1345    ObjectChanged,
1346}
1347
1348/// Failure while creating a SelectObjectContent snapshot.
1349#[derive(Debug, thiserror::Error)]
1350#[non_exhaustive]
1351pub enum PrepareSelectObjectSnapshotError {
1352    #[error("storage failed while preparing SelectObjectContent snapshot: {0}")]
1353    Storage(#[source] StorageError),
1354    #[error("SelectObjectContent snapshot consistency failure: {0}")]
1355    Consistency(#[source] SnapshotConsistencyError),
1356    #[error("SelectObjectContent object has invalid logical size {size}")]
1357    InvalidLogicalSize { size: i64 },
1358}
1359
1360impl From<StorageError> for PrepareSelectObjectSnapshotError {
1361    fn from(error: StorageError) -> Self {
1362        Self::Storage(error)
1363    }
1364}
1365
1366impl From<SnapshotConsistencyError> for PrepareSelectObjectSnapshotError {
1367    fn from(error: SnapshotConsistencyError) -> Self {
1368        Self::Consistency(error)
1369    }
1370}
1371
1372/// Failure while opening a reader from a SelectObjectContent snapshot.
1373#[derive(Debug, thiserror::Error)]
1374#[non_exhaustive]
1375pub enum SelectObjectSnapshotReadError {
1376    #[error("storage failed while opening SelectObjectContent snapshot reader: {0}")]
1377    Storage(#[source] StorageError),
1378    #[error("SelectObjectContent snapshot consistency failure: {0}")]
1379    Consistency(#[source] SnapshotConsistencyError),
1380}
1381
1382impl From<StorageError> for SelectObjectSnapshotReadError {
1383    fn from(error: StorageError) -> Self {
1384        Self::Storage(error)
1385    }
1386}
1387
1388impl From<SnapshotConsistencyError> for SelectObjectSnapshotReadError {
1389    fn from(error: SnapshotConsistencyError) -> Self {
1390        Self::Consistency(error)
1391    }
1392}
1393
1394struct SelectObjectSnapshotLease {
1395    guards: Vec<ObjectLockDiagGuard>,
1396    lost: Arc<AtomicBool>,
1397    lock_loss: tokio::sync::watch::Sender<bool>,
1398    _monitor_shutdown: tokio::sync::watch::Sender<()>,
1399}
1400
1401impl SelectObjectSnapshotLease {
1402    fn new(guards: Vec<ObjectLockDiagGuard>) -> Self {
1403        let signals = guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal);
1404        let mut waits = signals
1405            .map(|signal| async move { signal.notified().await })
1406            .collect::<futures::stream::FuturesUnordered<_>>();
1407        let lost = Arc::new(AtomicBool::new(false));
1408        let (lock_loss, _) = tokio::sync::watch::channel(false);
1409        let (monitor_shutdown, mut shutdown_rx) = tokio::sync::watch::channel(());
1410        if !waits.is_empty() {
1411            let task_lost = Arc::clone(&lost);
1412            let task_lock_loss = lock_loss.clone();
1413            tokio::spawn(async move {
1414                tokio::select! {
1415                    lost = futures::StreamExt::next(&mut waits) => {
1416                        if lost.is_some() {
1417                            task_lost.store(true, Ordering::Release);
1418                            task_lock_loss.send_replace(true);
1419                        }
1420                    }
1421                    _ = shutdown_rx.changed() => {}
1422                }
1423            });
1424        }
1425        Self {
1426            guards,
1427            lost,
1428            lock_loss,
1429            _monitor_shutdown: monitor_shutdown,
1430        }
1431    }
1432
1433    fn check(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1434        if self.is_lost() || self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
1435            self.lost.store(true, Ordering::Release);
1436            self.lock_loss.send_replace(true);
1437            return Err(SnapshotConsistencyError::LockLost);
1438        }
1439        Ok(())
1440    }
1441
1442    fn is_lost(&self) -> bool {
1443        self.lost.load(Ordering::Acquire)
1444    }
1445
1446    fn subscribe_lock_loss(&self) -> tokio::sync::watch::Receiver<bool> {
1447        self.lock_loss.subscribe()
1448    }
1449}
1450
1451/// Opaque, lock-backed object generation used by SelectObjectContent.
1452pub struct SelectObjectSnapshot {
1453    pool: Arc<Sets>,
1454    bucket: String,
1455    object: String,
1456    headers: HeaderMap,
1457    opts: ObjectOptions,
1458    object_info: ObjectInfo,
1459    logical_size: u64,
1460    read_semantics_identity: [u8; 32],
1461    first_metadata: ParkingMutex<Option<crate::set_disk::PreparedGetObjectMetadata>>,
1462    lease: Arc<SelectObjectSnapshotLease>,
1463}
1464
1465impl fmt::Debug for SelectObjectSnapshot {
1466    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1467        formatter
1468            .debug_struct("SelectObjectSnapshot")
1469            .field("bucket", &self.bucket)
1470            .field("object", &self.object)
1471            .field("logical_size", &self.logical_size)
1472            .finish_non_exhaustive()
1473    }
1474}
1475
1476impl SelectObjectSnapshot {
1477    pub fn is_for(&self, bucket: &str, object: &str) -> bool {
1478        self.bucket == bucket && self.object == encode_dir_object(object)
1479    }
1480
1481    pub fn object_info(&self) -> &ObjectInfo {
1482        &self.object_info
1483    }
1484
1485    pub fn logical_size(&self) -> u64 {
1486        self.logical_size
1487    }
1488
1489    pub fn matches_version(&self, requested: &str) -> bool {
1490        select_snapshot_version_matches(self.object_info.version_id, requested)
1491    }
1492
1493    pub fn ensure_valid(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1494        self.lease.check()
1495    }
1496
1497    #[cfg(feature = "test-util")]
1498    pub fn mark_lost_for_test(&self) {
1499        self.lease.lost.store(true, Ordering::Release);
1500        self.lease.lock_loss.send_replace(true);
1501    }
1502
1503    pub async fn open_reader(
1504        &self,
1505        range: Option<HTTPRangeSpec>,
1506    ) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
1507        self.lease.check()?;
1508        let first_metadata = self.first_metadata.lock().take();
1509        let metadata = match first_metadata {
1510            Some(metadata) => metadata,
1511            None => {
1512                self.pool
1513                    .prepare_get_object_reader_metadata(&self.bucket, &self.object, &self.opts)
1514                    .await?
1515            }
1516        };
1517        self.lease.check()?;
1518        if metadata.read_semantics_identity() != self.read_semantics_identity {
1519            return Err(SnapshotConsistencyError::ObjectChanged.into());
1520        }
1521
1522        let mut reader =
1523            crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
1524                &self.bucket,
1525                &self.object,
1526                range,
1527                self.headers.clone(),
1528                &self.opts,
1529                metadata,
1530            ))
1531            .await?;
1532        self.lease.check()?;
1533        reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
1534        reader.stream = Box::new(SelectObjectSnapshotReader {
1535            inner: reader.stream,
1536            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(self.lease.subscribe_lock_loss()),
1537            lease: Arc::clone(&self.lease),
1538        });
1539        Ok(reader)
1540    }
1541}
1542
1543fn select_snapshot_version_matches(actual: Option<Uuid>, requested: &str) -> bool {
1544    let requested = requested.trim();
1545    let requested = if requested.eq_ignore_ascii_case("null") {
1546        Uuid::nil()
1547    } else if let Ok(requested) = Uuid::parse_str(requested) {
1548        requested
1549    } else {
1550        return false;
1551    };
1552    actual.unwrap_or_else(Uuid::nil) == requested
1553}
1554
1555struct SelectObjectSnapshotReader {
1556    inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
1557    lock_loss_wake: SelectObjectSnapshotLockLossWake,
1558    lease: Arc<SelectObjectSnapshotLease>,
1559}
1560
1561struct SelectObjectSnapshotLockLossWake {
1562    stream: tokio_stream::wrappers::WatchStream<bool>,
1563}
1564
1565impl SelectObjectSnapshotLockLossWake {
1566    fn new(receiver: tokio::sync::watch::Receiver<bool>) -> Self {
1567        Self {
1568            stream: tokio_stream::wrappers::WatchStream::new(receiver),
1569        }
1570    }
1571
1572    fn poll_lost(&mut self, cx: &mut Context<'_>) -> bool {
1573        loop {
1574            match futures::Stream::poll_next(Pin::new(&mut self.stream), cx) {
1575                Poll::Ready(Some(true)) => return true,
1576                Poll::Ready(Some(false)) => {}
1577                Poll::Ready(None) | Poll::Pending => return false,
1578            }
1579        }
1580    }
1581}
1582
1583impl AsyncRead for SelectObjectSnapshotReader {
1584    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
1585        if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() {
1586            return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1587        }
1588        let filled_before = buf.filled().len();
1589        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1590        if self.lease.check().is_err() {
1591            buf.set_filled(filled_before);
1592            return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1593        }
1594        poll
1595    }
1596}
1597
1598fn log_object_lock_acquire_if_slow(
1599    op: &'static str,
1600    bucket: &str,
1601    object: &str,
1602    owner: Option<&str>,
1603    mode: ObjectLockDiagMode,
1604    elapsed: Duration,
1605    diag_enabled: bool,
1606) {
1607    if !diag_enabled {
1608        return;
1609    }
1610
1611    let threshold = get_object_lock_diag_slow_acquire_threshold();
1612    record_object_lock_diag_acquire_duration(op, mode.as_str(), elapsed);
1613    if elapsed >= threshold {
1614        record_object_lock_diag_slow_acquire(op, mode.as_str());
1615        warn!(
1616            target: "rustfs_ecstore::object_lock_diag",
1617            op,
1618            bucket,
1619            object,
1620            mode = %mode,
1621            owner = owner.unwrap_or_default(),
1622            acquire_ms = elapsed.as_millis(),
1623            threshold_ms = threshold.as_millis(),
1624            "object namespace lock acquisition exceeded threshold"
1625        );
1626    }
1627}
1628
1629fn select_data_movement_target_pool(
1630    existing_pool_idx: Result<usize>,
1631    src_pool_idx: usize,
1632    delete_marker: bool,
1633) -> Result<Option<usize>> {
1634    match existing_pool_idx {
1635        Ok(pool_idx) => {
1636            if delete_marker && pool_idx == src_pool_idx {
1637                Ok(None)
1638            } else {
1639                Ok(Some(pool_idx))
1640            }
1641        }
1642        Err(err) => {
1643            if is_err_read_quorum(&err) {
1644                return Err(StorageError::ErasureWriteQuorum);
1645            }
1646            if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
1647                Ok(None)
1648            } else {
1649                Err(err)
1650            }
1651        }
1652    }
1653}
1654
1655fn latest_object_access_delete_marker_error(
1656    bucket: &str,
1657    object: &str,
1658    info: &ObjectInfo,
1659    opts: &ObjectOptions,
1660) -> Option<Error> {
1661    if !info.delete_marker {
1662        return None;
1663    }
1664
1665    Some(if opts.version_id.is_none() || opts.delete_marker {
1666        to_object_err(StorageError::FileNotFound, vec![bucket, object])
1667    } else {
1668        to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])
1669    })
1670}
1671
1672fn resolve_latest_object_access(
1673    bucket: &str,
1674    object: &str,
1675    info: ObjectInfo,
1676    idx: usize,
1677    opts: &ObjectOptions,
1678) -> Result<(ObjectInfo, usize)> {
1679    if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) {
1680        return Err(err);
1681    }
1682
1683    Ok((info, idx))
1684}
1685
1686fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
1687    (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
1688}
1689
1690fn latest_versioned_delete_creates_distinct_marker(opts: &ObjectOptions) -> bool {
1691    opts.versioned
1692        && !opts.version_suspended
1693        && opts.version_id.is_none()
1694        && !opts.delete_marker
1695        && !opts.data_movement
1696        && !opts.replication_request
1697}
1698
1699#[cfg(any(test, feature = "test-util"))]
1700struct DeleteAfterObjectLockSnapshotBarrierState {
1701    bucket: String,
1702    arrived: tokio::sync::Notify,
1703    release: tokio::sync::Notify,
1704    namespace_pending: tokio::sync::Notify,
1705    namespace_acquired: AtomicBool,
1706}
1707
1708#[cfg(any(test, feature = "test-util"))]
1709pub struct DeleteAfterObjectLockSnapshotBarrier {
1710    state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
1711}
1712
1713#[cfg(any(test, feature = "test-util"))]
1714static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
1715    std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
1716> = std::sync::OnceLock::new();
1717
1718#[cfg(any(test, feature = "test-util"))]
1719impl DeleteAfterObjectLockSnapshotBarrier {
1720    pub fn install(bucket: &str) -> Self {
1721        let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
1722            bucket: bucket.to_string(),
1723            arrived: tokio::sync::Notify::new(),
1724            release: tokio::sync::Notify::new(),
1725            namespace_pending: tokio::sync::Notify::new(),
1726            namespace_acquired: AtomicBool::new(false),
1727        });
1728        let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1729            .get_or_init(|| std::sync::Mutex::new(None))
1730            .lock()
1731            .expect("delete snapshot barrier mutex should not poison");
1732        assert!(slot.is_none(), "delete snapshot barrier must not already be installed");
1733        *slot = Some(Arc::clone(&state));
1734        Self { state }
1735    }
1736
1737    pub async fn wait_until_paused(&self) {
1738        self.state.arrived.notified().await;
1739    }
1740
1741    pub fn release(&self) {
1742        self.state.release.notify_one();
1743    }
1744
1745    pub async fn release_and_wait_until_namespace_pending(&self) {
1746        let namespace_pending = self.state.namespace_pending.notified();
1747        self.release();
1748        tokio::time::timeout(Duration::from_secs(5), namespace_pending)
1749            .await
1750            .expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
1751    }
1752
1753    pub fn namespace_acquired(&self) -> bool {
1754        self.state.namespace_acquired.load(Ordering::Acquire)
1755    }
1756}
1757
1758#[cfg(any(test, feature = "test-util"))]
1759impl Drop for DeleteAfterObjectLockSnapshotBarrier {
1760    fn drop(&mut self) {
1761        self.state.release.notify_one();
1762        if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() {
1763            let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison");
1764            if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
1765                *slot = None;
1766            }
1767        }
1768    }
1769}
1770
1771#[cfg(any(test, feature = "test-util"))]
1772async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
1773    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1774        .get_or_init(|| std::sync::Mutex::new(None))
1775        .lock()
1776        .expect("delete snapshot barrier mutex should not poison")
1777        .as_ref()
1778        .filter(|state| state.bucket == bucket)
1779        .cloned();
1780    if let Some(state) = state {
1781        state.arrived.notify_one();
1782        state.release.notified().await;
1783    }
1784}
1785
1786#[cfg(any(test, feature = "test-util"))]
1787fn notify_delete_namespace_pending(bucket: &str) {
1788    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1789        .get_or_init(|| std::sync::Mutex::new(None))
1790        .lock()
1791        .expect("delete snapshot barrier mutex should not poison")
1792        .as_ref()
1793        .filter(|state| state.bucket == bucket)
1794        .cloned();
1795    if let Some(state) = state {
1796        state.namespace_pending.notify_one();
1797    }
1798}
1799
1800#[cfg(any(test, feature = "test-util"))]
1801fn notify_delete_namespace_acquired(bucket: &str) {
1802    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1803        .get_or_init(|| std::sync::Mutex::new(None))
1804        .lock()
1805        .expect("delete snapshot barrier mutex should not poison")
1806        .as_ref()
1807        .filter(|state| state.bucket == bucket)
1808        .cloned();
1809    if let Some(state) = state {
1810        state.namespace_acquired.store(true, Ordering::Release);
1811    }
1812}
1813
1814#[cfg(test)]
1815struct VersionedDeleteMarkerCommitBarrierState {
1816    bucket: String,
1817    object: String,
1818    arrived: tokio::sync::Notify,
1819    release: tokio::sync::Notify,
1820}
1821
1822#[cfg(test)]
1823pub(crate) struct VersionedDeleteMarkerCommitBarrier {
1824    state: Arc<VersionedDeleteMarkerCommitBarrierState>,
1825}
1826
1827#[cfg(test)]
1828static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock<
1829    std::sync::Mutex<Option<Arc<VersionedDeleteMarkerCommitBarrierState>>>,
1830> = std::sync::OnceLock::new();
1831
1832#[cfg(test)]
1833impl VersionedDeleteMarkerCommitBarrier {
1834    pub(crate) fn install(bucket: &str, object: &str) -> Self {
1835        let state = Arc::new(VersionedDeleteMarkerCommitBarrierState {
1836            bucket: bucket.to_string(),
1837            object: object.to_string(),
1838            arrived: tokio::sync::Notify::new(),
1839            release: tokio::sync::Notify::new(),
1840        });
1841        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1842            .get_or_init(|| std::sync::Mutex::new(None))
1843            .lock()
1844            .expect("versioned delete-marker commit barrier mutex should not poison");
1845        assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique");
1846        *slot = Some(Arc::clone(&state));
1847        Self { state }
1848    }
1849
1850    pub(crate) async fn wait_until_paused(&self) {
1851        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1852            .await
1853            .expect("versioned DELETE should reach the post-marker-commit barrier");
1854    }
1855
1856    pub(crate) fn release(&self) {
1857        self.state.release.notify_one();
1858    }
1859}
1860
1861#[cfg(test)]
1862impl Drop for VersionedDeleteMarkerCommitBarrier {
1863    fn drop(&mut self) {
1864        self.state.release.notify_one();
1865        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1866            .get_or_init(|| std::sync::Mutex::new(None))
1867            .lock()
1868            .expect("versioned delete-marker commit barrier mutex should not poison");
1869        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1870            *slot = None;
1871        }
1872    }
1873}
1874
1875#[cfg(test)]
1876async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) {
1877    let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1878        .get_or_init(|| std::sync::Mutex::new(None))
1879        .lock()
1880        .expect("versioned delete-marker commit barrier mutex should not poison")
1881        .as_ref()
1882        .filter(|state| state.bucket == bucket && state.object == object)
1883        .cloned();
1884    if let Some(state) = state {
1885        state.arrived.notify_one();
1886        state.release.notified().await;
1887    }
1888}
1889
1890/// Whether a delete-time lookup miss on a directory key should trigger an orphan
1891/// empty-directory tree purge (issue #4189).
1892///
1893/// The lookup surfaces *version*-not-found here, not object-not-found: `del_opts`
1894/// pins `version_id = Uuid::nil()` for directory keys, so a missing dir object fails
1895/// the specific-version lookup. Both misses must be accepted, otherwise the real
1896/// HTTP delete path (which always sets the nil version) never reaches the purge and
1897/// the ghost folder survives with a fake 204 — the exact #4189 symptom.
1898fn should_purge_orphan_dir_on_missing(err: &Error, object: &str) -> bool {
1899    (is_err_object_not_found(err) || is_err_version_not_found(err)) && rustfs_utils::path::is_dir_object(object)
1900}
1901
1902fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1903    let mut lookup_opts = opts.clone();
1904    lookup_opts.no_lock = no_lock;
1905    if lookup_opts.version_id.is_some() {
1906        lookup_opts.metadata_chg = true;
1907    }
1908
1909    lookup_opts
1910}
1911
1912fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1913    writer_pool_lookup_opts(opts, no_lock)
1914}
1915
1916fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool {
1917    opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some())
1918}
1919
1920fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1921    let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
1922    lookup_opts.skip_decommissioned = true;
1923    lookup_opts.skip_rebalancing = true;
1924
1925    lookup_opts
1926}
1927
1928fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1929    let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock);
1930    lookup_opts.skip_decommissioned = opts.data_movement;
1931    lookup_opts
1932}
1933
1934fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
1935    pool_count > 0 && delete_only_releases_capacity(opts)
1936}
1937
1938fn delete_only_releases_capacity(opts: &ObjectOptions) -> bool {
1939    !opts.versioned && !opts.version_suspended || opts.version_id.is_some()
1940}
1941
1942fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
1943    if object.version_id.is_some() {
1944        return false;
1945    }
1946
1947    let object_name = decode_dir_object(&object.object_name);
1948    let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name);
1949    versioned || version_suspended
1950}
1951
1952fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option<usize>, pool_idx: usize) -> bool {
1953    !creates_latest_marker || marker_target_pool_idx == Some(pool_idx)
1954}
1955
1956#[cfg(test)]
1957struct BatchDeletePoolErrorInjectionState {
1958    bucket: String,
1959    pool_idx: usize,
1960    errors: std::collections::HashMap<String, Error>,
1961    observed: std::sync::atomic::AtomicUsize,
1962}
1963
1964#[cfg(test)]
1965pub(crate) struct BatchDeletePoolErrorInjection {
1966    state: Arc<BatchDeletePoolErrorInjectionState>,
1967}
1968
1969#[cfg(test)]
1970static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock<std::sync::Mutex<Option<Arc<BatchDeletePoolErrorInjectionState>>>> =
1971    std::sync::OnceLock::new();
1972
1973#[cfg(test)]
1974impl BatchDeletePoolErrorInjection {
1975    pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self {
1976        let state = Arc::new(BatchDeletePoolErrorInjectionState {
1977            bucket: bucket.to_string(),
1978            pool_idx,
1979            errors: errors.into_iter().collect(),
1980            observed: std::sync::atomic::AtomicUsize::new(0),
1981        });
1982        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
1983            .get_or_init(|| std::sync::Mutex::new(None))
1984            .lock()
1985            .expect("batch delete pool error injection mutex should not poison");
1986        assert!(slot.is_none(), "batch delete pool error injection must be unique");
1987        *slot = Some(Arc::clone(&state));
1988        Self { state }
1989    }
1990
1991    pub(crate) fn observed(&self) -> usize {
1992        self.state.observed.load(Ordering::Acquire)
1993    }
1994}
1995
1996#[cfg(test)]
1997impl Drop for BatchDeletePoolErrorInjection {
1998    fn drop(&mut self) {
1999        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
2000            .get_or_init(|| std::sync::Mutex::new(None))
2001            .lock()
2002            .expect("batch delete pool error injection mutex should not poison");
2003        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
2004            *slot = None;
2005        }
2006    }
2007}
2008
2009#[cfg(test)]
2010fn inject_batch_delete_pool_errors(
2011    bucket: &str,
2012    pool_idx: usize,
2013    object_names: &[String],
2014    deleted: &[DeletedObject],
2015    errors: &mut [Option<Error>],
2016) {
2017    let state = BATCH_DELETE_POOL_ERROR_INJECTION
2018        .get_or_init(|| std::sync::Mutex::new(None))
2019        .lock()
2020        .expect("batch delete pool error injection mutex should not poison")
2021        .as_ref()
2022        .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx)
2023        .cloned();
2024    let Some(state) = state else {
2025        return;
2026    };
2027
2028    for (idx, object_name) in object_names.iter().enumerate() {
2029        let Some(error) = state.errors.get(object_name) else {
2030            continue;
2031        };
2032        if errors[idx].is_none() && deleted[idx].found {
2033            errors[idx] = Some(error.clone());
2034            state.observed.fetch_add(1, Ordering::AcqRel);
2035        }
2036    }
2037}
2038
2039fn resolve_batch_delete_pool_results<'a>(
2040    initial_error: Option<Error>,
2041    pool_results: impl IntoIterator<Item = (&'a DeletedObject, &'a Option<Error>)>,
2042) -> (Option<DeletedObject>, Option<Error>, bool) {
2043    let mut failure = initial_error.map(|err| (None, err));
2044    let mut deleted = None;
2045    let mut fallback: Option<(DeletedObject, Option<Error>)> = None;
2046    let mut attempted = false;
2047
2048    for (pool_delete, pool_error) in pool_results {
2049        attempted = true;
2050        match pool_error {
2051            Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => {
2052                if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) {
2053                    fallback = Some(((*pool_delete).clone(), Some(err.clone())));
2054                }
2055            }
2056            Some(err) => {
2057                if failure.is_none() {
2058                    failure = Some((Some((*pool_delete).clone()), err.clone()));
2059                }
2060            }
2061            None if pool_delete.found => {
2062                if deleted.is_none() {
2063                    deleted = Some((*pool_delete).clone());
2064                }
2065            }
2066            None => {
2067                if fallback.is_none() {
2068                    fallback = Some(((*pool_delete).clone(), None));
2069                }
2070            }
2071        }
2072    }
2073
2074    if let Some((failed_delete, err)) = failure {
2075        return (failed_delete, Some(err), attempted);
2076    }
2077    if let Some(deleted) = deleted {
2078        return (Some(deleted), None, attempted);
2079    }
2080    if let Some((deleted, err)) = fallback {
2081        return (Some(deleted), err, attempted);
2082    }
2083
2084    (None, None, attempted)
2085}
2086
2087fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
2088    let mut lookup_opts = opts.clone();
2089    // `no_lock` is an internal implementation detail, not transferable
2090    // authority. Restore deliberately does not hold an outer object write lock,
2091    // so every final local commit must acquire its own commit-late lock.
2092    lookup_opts.no_lock = false;
2093    lookup_opts.skip_decommissioned = true;
2094    lookup_opts.skip_rebalancing = true;
2095    lookup_opts
2096}
2097
2098fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
2099    info.get_actual_size().ok()
2100}
2101
2102pub(super) fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2103    is_data_movement_delete_marker(source)
2104        && is_data_movement_delete_marker(target)
2105        && source.version_id == target.version_id
2106        && source.mod_time == target.mod_time
2107        && is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
2108        && source.user_tags == target.user_tags
2109        && source.replication_status_internal == target.replication_status_internal
2110        && source.replication_status == target.replication_status
2111        && source.version_purge_status_internal == target.version_purge_status_internal
2112        && source.version_purge_status == target.version_purge_status
2113}
2114
2115fn is_equivalent_data_movement_delete_marker_metadata(
2116    source: &HashMap<String, String>,
2117    target: &HashMap<String, String>,
2118) -> bool {
2119    matches!(
2120        (
2121            data_movement_delete_marker_metadata_identity(source),
2122            data_movement_delete_marker_metadata_identity(target)
2123        ),
2124        (Some(source), Some(target)) if source == target
2125    )
2126}
2127
2128fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
2129    let mut identity = HashMap::with_capacity(metadata.len());
2130    let mut local_tier_free_version_id = None;
2131    for (key, value) in metadata {
2132        let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
2133            identity.insert(key.clone(), value.clone());
2134            continue;
2135        };
2136
2137        if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
2138            let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
2139            if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
2140                return None;
2141            }
2142            local_tier_free_version_id = Some(version_id);
2143            continue;
2144        }
2145        if suffix.eq_ignore_ascii_case(rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID) {
2146            continue;
2147        }
2148
2149        let canonical_suffix = [
2150            rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
2151            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
2152            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2153            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2154            rustfs_utils::http::SUFFIX_PURGESTATUS,
2155        ]
2156        .into_iter()
2157        .find(|candidate| suffix.eq_ignore_ascii_case(candidate))
2158        .map(str::to_string)
2159        .or_else(|| {
2160            [
2161                rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
2162                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
2163            ]
2164            .into_iter()
2165            .find_map(|prefix| {
2166                suffix
2167                    .get(..prefix.len())
2168                    .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
2169                    .then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
2170            })
2171        })
2172        .unwrap_or_else(|| suffix.to_string());
2173        let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
2174            || canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
2175        {
2176            rustfs_filemeta::parse_replication_timestamp(value)?
2177                .unix_timestamp_nanos()
2178                .to_string()
2179        } else {
2180            value.clone()
2181        };
2182        let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2183        if identity
2184            .insert(canonical_key, canonical_value.clone())
2185            .is_some_and(|existing| existing != canonical_value)
2186        {
2187            return None;
2188        }
2189    }
2190    for (status_suffix, timestamp_suffix) in [
2191        (rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
2192        (
2193            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2194            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2195        ),
2196    ] {
2197        let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2198        let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2199        match (identity.contains_key(&status_key), identity.contains_key(&timestamp_key)) {
2200            (true, false) => {
2201                identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
2202            }
2203            (false, true) => return None,
2204            _ => {}
2205        }
2206    }
2207    Some(identity)
2208}
2209
2210fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
2211    info.delete_marker
2212}
2213
2214fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
2215    is_data_movement_delete_marker(source)
2216        && source.mod_time.is_some()
2217        && source.mod_time == expected_mod_time
2218        && data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
2219}
2220
2221fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
2222    let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
2223    let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
2224    let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
2225        (None, None) => Default::default(),
2226        (Some(status), timestamp) => {
2227            let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2228            if status.is_empty() {
2229                return None;
2230            }
2231            let timestamp = match timestamp {
2232                Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
2233                None => OffsetDateTime::UNIX_EPOCH,
2234            };
2235            (status, Some(timestamp))
2236        }
2237        (None, Some(_)) => return None,
2238    };
2239    let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
2240    let replication_timestamp =
2241        rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
2242    let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
2243        (None, None) => Default::default(),
2244        (Some(status), timestamp) => {
2245            let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2246            let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
2247            if direct_status.is_empty() && targets.is_empty() {
2248                return None;
2249            }
2250            let timestamp = match timestamp {
2251                Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
2252                None => OffsetDateTime::UNIX_EPOCH,
2253            };
2254            (Some(status), Some(timestamp), targets)
2255        }
2256        (None, Some(_)) => return None,
2257    };
2258    let mut state = source.replication_state();
2259    if state.target_delete_marker_version_ids_corrupt {
2260        return None;
2261    }
2262    state.replica_status = replica_status;
2263    state.replica_timestamp = replica_timestamp;
2264    state.replication_status_internal = replication_status;
2265    state.replication_timestamp = replication_timestamp;
2266    state.targets = replication_targets;
2267    state.replicate_decision_str = source.replication_decision.clone();
2268    state.delete_marker = true;
2269
2270    let mut target_opts = opts.clone();
2271    target_opts.mod_time = source.mod_time;
2272    target_opts.delete_replication = Some(state);
2273    Some(target_opts)
2274}
2275
2276fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
2277    ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
2278}
2279
2280fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
2281    let expected = expected_data_movement_tiered_object(source);
2282    let Some(source_actual_size) = effective_object_actual_size(&expected) else {
2283        return false;
2284    };
2285    let Some(target_actual_size) = effective_object_actual_size(target) else {
2286        return false;
2287    };
2288
2289    source.version_id == target.version_id
2290        && !target.delete_marker
2291        && source.size == target.size
2292        && source.get_etag() == target.etag
2293        && source.checksum == target.checksum
2294        && crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
2295        && source.mod_time == target.mod_time
2296        && crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
2297        && expected.user_tags == target.user_tags
2298        && expected.expires == target.expires
2299        && expected.storage_class == target.storage_class
2300        && expected.replication_status_internal == target.replication_status_internal
2301        && expected.replication_status == target.replication_status
2302        && expected.version_purge_status_internal == target.version_purge_status_internal
2303        && expected.version_purge_status == target.version_purge_status
2304        && expected.transitioned_object.status == target.transitioned_object.status
2305        && expected.transition_version_state == target.transition_version_state
2306        && expected.transitioned_object.name == target.transitioned_object.name
2307        && expected.transitioned_object.tier == target.transitioned_object.tier
2308        && expected.transitioned_object.version_id == target.transitioned_object.version_id
2309        && expected.transitioned_object.free_version == target.transitioned_object.free_version
2310        && source_actual_size == target_actual_size
2311}
2312
2313pub(crate) fn tiered_data_movement_source_matches(
2314    expected: &rustfs_filemeta::FileInfo,
2315    current: &rustfs_filemeta::FileInfo,
2316) -> Result<bool> {
2317    let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
2318    let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&current.metadata)?;
2319    Ok(expected.version_id == current.version_id
2320        && expected.deleted == current.deleted
2321        && expected.tier_free_version() == current.tier_free_version()
2322        && expected.data_dir == current.data_dir
2323        && expected.mod_time == current.mod_time
2324        && expected.size == current.size
2325        && expected.get_etag() == current.get_etag()
2326        && expected.transition_status == current.transition_status
2327        && expected.transitioned_objname == current.transitioned_objname
2328        && expected.transition_tier == current.transition_tier
2329        && expected.transition_version_id == current.transition_version_id
2330        && expected.transition_version == current.transition_version
2331        && expected.transition_version_state == current.transition_version_state
2332        && expected_backend == current_backend)
2333}
2334
2335fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
2336    StorageError::DataMovementOverwriteErr(
2337        bucket.to_owned(),
2338        object.to_owned(),
2339        version_id.map(|id| id.to_string()).unwrap_or_default(),
2340    )
2341}
2342
2343fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
2344    target_pool_idx != src_pool_idx
2345}
2346
2347fn resolve_data_movement_resume_target_pool(
2348    selected_target_pool_idx: usize,
2349    resume_target_pool_idx: Option<usize>,
2350    src_pool_idx: usize,
2351) -> usize {
2352    if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
2353        selected_target_pool_idx
2354    } else {
2355        resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
2356    }
2357}
2358
2359fn resolve_data_movement_delete_marker_resume_result(
2360    target_result: Result<Option<ObjectInfo>>,
2361    source: &ObjectInfo,
2362    src_pool_idx: usize,
2363    target_pool_idx: usize,
2364) -> Result<bool> {
2365    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2366        return Ok(false);
2367    }
2368
2369    let Some(target) = target_result? else {
2370        return Ok(false);
2371    };
2372
2373    Ok(is_equivalent_data_movement_delete_marker(source, &target))
2374}
2375
2376fn resolve_data_movement_tiered_resume_result(
2377    target_result: Result<Option<ObjectInfo>>,
2378    source: &rustfs_filemeta::FileInfo,
2379    src_pool_idx: usize,
2380    target_pool_idx: usize,
2381) -> Result<bool> {
2382    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2383        return Ok(false);
2384    }
2385
2386    let Some(target) = target_result? else {
2387        return Ok(false);
2388    };
2389
2390    Ok(is_equivalent_data_movement_tiered_object(source, &target))
2391}
2392
2393fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
2394    let del_objects = objects
2395        .iter()
2396        .map(|object| DeletedObject {
2397            object_name: decode_dir_object(&object.object_name),
2398            version_id: object.version_id,
2399            ..Default::default()
2400        })
2401        .collect();
2402    let del_errs = objects.iter().map(|_| Some(err.clone())).collect();
2403
2404    (del_objects, del_errs)
2405}
2406
2407fn return_batch_delete_lock_error_with_accounting(
2408    objects: &[ObjectToDelete],
2409    err: Error,
2410) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
2411    let (deleted, errors) = return_batch_delete_lock_error(objects, err);
2412    (deleted, errors, vec![None; objects.len()])
2413}
2414
2415fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
2416    let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
2417    object_names.sort_unstable();
2418    object_names.dedup();
2419    object_names
2420}
2421
2422/// A CopyObject branch that writes a fresh local data stream owns new bytes and
2423/// must not inherit the source version's remote-tier ownership record.  FileMeta
2424/// exposes internal transition fields through `ObjectInfo::user_defined` for
2425/// compatibility, so cloning that map verbatim would publish a second owner of
2426/// the same remote tuple even though the destination has local data.
2427///
2428/// Keep this normalization in the store layer so every full-copy branch in
2429/// `handle_copy_object` shares the same fail-safe boundary. Metadata-only and
2430/// version-only same-key copies do not use this helper and continue to preserve
2431/// the existing version's protected state.
2432fn materialized_copy_user_defined(source: &HashMap<String, String>) -> HashMap<String, String> {
2433    let mut metadata = source.clone();
2434    for suffix in [
2435        rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2436        rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2437        rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2438        rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2439        rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2440        rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2441        rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2442        rustfs_utils::http::SUFFIX_FREE_VERSION,
2443        rustfs_utils::http::SUFFIX_TIER_FV_ID,
2444        rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
2445    ] {
2446        rustfs_utils::http::metadata_compat::remove_str(&mut metadata, suffix);
2447    }
2448    metadata
2449}
2450
2451enum RemoteTuplePublicationSource {
2452    Object(ObjectInfo),
2453    Tiered(rustfs_filemeta::FileInfo),
2454}
2455
2456/// A decommission migration may retain the fixed store domain as a read
2457/// anchor while it streams bytes. This type cannot authorize publication: it
2458/// must be consumed and dropped before the commit path acquires the fixed
2459/// domain as a write lock.
2460#[must_use = "the fixed read anchor must remain live until publication or migration abort"]
2461pub(crate) struct DecommissionFixedReadAnchor {
2462    guard: ObjectLockDiagGuard,
2463}
2464
2465impl DecommissionFixedReadAnchor {
2466    pub(crate) fn guard(&self) -> &ObjectLockDiagGuard {
2467        &self.guard
2468    }
2469
2470    fn is_lock_lost(&self) -> bool {
2471        self.guard.is_lock_lost()
2472    }
2473}
2474
2475/// Opaque, non-cloneable capability retained by every background path that can
2476/// publish an existing object identity into a new physical metadata owner.
2477///
2478/// The exact tier destination is captured before the copy, but its operation
2479/// lease and namespace write locks are deliberately deferred until the final
2480/// metadata commit. The source is then re-read under the same lock set. This
2481/// keeps large migrations compatible with the commit-late locking contract and
2482/// avoids pinning a tier generation throughout multipart staging.
2483#[must_use = "a data-movement publication fence must be consumed by the final target commit"]
2484pub(crate) struct RemoteTuplePublicationFence {
2485    store: Arc<ECStore>,
2486    bucket: String,
2487    object: String,
2488    source_pool_idx: usize,
2489    source: RemoteTuplePublicationSource,
2490    include_fixed_domain: bool,
2491    fixed_read_anchor: Option<DecommissionFixedReadAnchor>,
2492    backend_target: Option<(String, Option<TierDestinationId>)>,
2493}
2494
2495/// Locks and tier generation retained from source revalidation through the
2496/// target rename quorum (and any rename-tail guard handoff).
2497#[must_use = "the publication commit guard must live through the target metadata commit"]
2498pub(crate) struct RemoteTuplePublicationCommitGuard {
2499    guards: Vec<ObjectLockDiagGuard>,
2500    backend_lease: Option<TierOperationLease>,
2501    revalidated_tiered_source: Option<rustfs_filemeta::FileInfo>,
2502}
2503
2504impl RemoteTuplePublicationCommitGuard {
2505    pub(crate) fn is_lock_lost(&self) -> bool {
2506        self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2507            || self
2508                .backend_lease
2509                .as_ref()
2510                .is_some_and(|lease| !lease.is_current_generation())
2511    }
2512
2513    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
2514        for guard in &self.guards {
2515            guard.add_namespace_lock_fence(opts);
2516        }
2517    }
2518
2519    pub(crate) fn revalidated_tiered_source(&self) -> Option<&rustfs_filemeta::FileInfo> {
2520        self.revalidated_tiered_source.as_ref()
2521    }
2522}
2523
2524fn remote_tuple_publication_target(source: &ObjectInfo) -> std::io::Result<Option<(String, Option<TierDestinationId>)>> {
2525    if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !source.transitioned_object.free_version {
2526        return Ok(None);
2527    }
2528    if source.transitioned_object.tier.is_empty() || source.transitioned_object.name.is_empty() {
2529        return Err(std::io::Error::other(
2530            "remote tuple publication source is missing its tier or remote object identity",
2531        ));
2532    }
2533    let backend_identity = tier_destination_id_from_metadata(source.user_defined.as_ref())?;
2534    Ok(Some((source.transitioned_object.tier.clone(), backend_identity)))
2535}
2536
2537fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current: &ObjectInfo) -> bool {
2538    let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else {
2539        return false;
2540    };
2541
2542    expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil())
2543        && expected.data_dir == current.data_dir
2544        && expected.version_id == current.version_id
2545        && expected.delete_marker == current.delete_marker
2546        && expected.size == current.size
2547        && expected_actual_size == current_actual_size
2548        && expected.etag == current.etag
2549        && expected.checksum == current.checksum
2550        && expected.mod_time == current.mod_time
2551        && expected.storage_class == current.storage_class
2552        && crate::data_movement::is_equivalent_data_movement_metadata(
2553            expected,
2554            current,
2555            expected_actual_size,
2556            current_actual_size,
2557        )
2558        && expected.user_tags == current.user_tags
2559        && expected.expires == current.expires
2560        && expected.replication_status_internal == current.replication_status_internal
2561        && expected.replication_status == current.replication_status
2562        && expected.version_purge_status_internal == current.version_purge_status_internal
2563        && expected.version_purge_status == current.version_purge_status
2564        && expected.transitioned_object.name == current.transitioned_object.name
2565        && expected.transitioned_object.version_id == current.transitioned_object.version_id
2566        && expected.transitioned_object.tier == current.transitioned_object.tier
2567        && expected.transitioned_object.free_version == current.transitioned_object.free_version
2568        && expected.transitioned_object.status == current.transitioned_object.status
2569        && expected.transition_version_state == current.transition_version_state
2570        && remote_tuple_publication_parts_match(&expected.parts, &current.parts)
2571}
2572
2573fn remote_tuple_publication_parts_match(expected: &[ObjectPartInfo], current: &[ObjectPartInfo]) -> bool {
2574    if expected.len() != current.len() {
2575        return false;
2576    }
2577    let Some(mut current_parts) = crate::data_movement::data_movement_parts_by_number(current) else {
2578        return false;
2579    };
2580
2581    expected.iter().all(|expected_part| {
2582        current_parts.remove(&expected_part.number).is_some_and(|current_part| {
2583            current_part.size == expected_part.size
2584                && current_part.actual_size == expected_part.actual_size
2585                && current_part.etag == expected_part.etag
2586        })
2587    })
2588}
2589
2590impl RemoteTuplePublicationFence {
2591    pub(crate) fn under_fixed_read_anchor(mut self, anchor: DecommissionFixedReadAnchor) -> Result<Self> {
2592        if self.include_fixed_domain {
2593            return Err(Error::other(
2594                "data movement publication capability cannot attach a fixed read anchor after requesting fixed write",
2595            ));
2596        }
2597        if anchor.is_lock_lost() {
2598            return Err(Error::other(
2599                "data movement publication capability received an already-lost fixed read anchor",
2600            ));
2601        }
2602        self.include_fixed_domain = self.backend_target.is_some();
2603        self.fixed_read_anchor = Some(anchor);
2604        Ok(self)
2605    }
2606
2607    pub(crate) fn fixed_read_anchor_guard(&self) -> Option<&ObjectLockDiagGuard> {
2608        self.fixed_read_anchor.as_ref().map(DecommissionFixedReadAnchor::guard)
2609    }
2610
2611    pub(crate) async fn into_commit_guard(
2612        self,
2613        target_pool_idx: usize,
2614        bucket: &str,
2615        object: &str,
2616    ) -> Result<RemoteTuplePublicationCommitGuard> {
2617        let Self {
2618            store,
2619            bucket: expected_bucket,
2620            object: expected_object,
2621            source_pool_idx,
2622            source,
2623            include_fixed_domain,
2624            fixed_read_anchor,
2625            backend_target,
2626        } = self;
2627        if bucket != expected_bucket || object != expected_object {
2628            return Err(Error::other_with_context(
2629                "data movement publication capability target mismatch",
2630                format!("expected {expected_bucket}/{expected_object}, got {bucket}/{object}"),
2631            ));
2632        }
2633        if fixed_read_anchor
2634            .as_ref()
2635            .is_some_and(DecommissionFixedReadAnchor::is_lock_lost)
2636        {
2637            return Err(Error::other("data movement publication fixed read anchor was lost before commit"));
2638        }
2639        // Never upgrade a held read lock in place. Releasing here keeps the
2640        // long transfer on a read anchor while the commit obtains a short
2641        // fixed-domain write lock in the global tier -> namespace order.
2642        drop(fixed_read_anchor);
2643        // Tier generation is acquired at the commit boundary, before any
2644        // namespace lock, preserving bucket -> tier -> namespace lock order.
2645        let backend_lease = if let Some((tier_name, backend_identity)) = backend_target {
2646            let manager = store.tier_config_mgr();
2647            Some(
2648                match backend_identity {
2649                    Some(identity) => {
2650                        TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, &tier_name, identity).await
2651                    }
2652                    None => TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await,
2653                }
2654                .map_err(Error::other)?,
2655            )
2656        } else {
2657            None
2658        };
2659
2660        let guards = store
2661            .acquire_data_movement_publication_write_locks(bucket, object, source_pool_idx, target_pool_idx, include_fixed_domain)
2662            .await?;
2663        if guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2664            || backend_lease.as_ref().is_some_and(|lease| !lease.is_current_generation())
2665        {
2666            return Err(Error::other("data movement publication fence was lost before source revalidation"));
2667        }
2668
2669        let source_pool = store.pools.get(source_pool_idx).ok_or_else(|| {
2670            Error::other_with_context("invalid data movement source pool", format!("pool index {source_pool_idx}"))
2671        })?;
2672        let mut revalidated_tiered_source = None;
2673        let source_matches = match source {
2674            RemoteTuplePublicationSource::Object(expected) => {
2675                let lookup_opts = ObjectOptions {
2676                    versioned: expected.version_id.is_some(),
2677                    version_id: expected.version_id.map(|version_id| version_id.to_string()),
2678                    incl_free_versions: expected.transitioned_object.free_version,
2679                    include_part_checksums: true,
2680                    no_lock: true,
2681                    ..Default::default()
2682                };
2683                match source_pool.get_object_info(bucket, object, &lookup_opts).await {
2684                    Ok(current) => remote_tuple_publication_object_source_matches(&expected, &current),
2685                    Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => false,
2686                    Err(err) => return Err(err),
2687                }
2688            }
2689            RemoteTuplePublicationSource::Tiered(expected) => {
2690                let logical_object = decode_dir_object(object);
2691                let current_versions = source_pool
2692                    .get_disks_by_key(object)
2693                    .load_file_info_versions_exact(bucket, &logical_object)
2694                    .await?;
2695                let is_free_version = expected.tier_free_version();
2696                let Some(current) = current_versions.as_ref().and_then(|versions| {
2697                    versions.versions.iter().find(|current| {
2698                        current.version_id == expected.version_id && current.tier_free_version() == is_free_version
2699                    })
2700                }) else {
2701                    return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
2702                };
2703                if tiered_data_movement_source_matches(&expected, current)? {
2704                    revalidated_tiered_source = Some(current.clone());
2705                    true
2706                } else {
2707                    false
2708                }
2709            }
2710        };
2711        if !source_matches {
2712            return Err(StorageError::DataMovementOverwriteErr(
2713                bucket.to_string(),
2714                object.to_string(),
2715                String::new(),
2716            ));
2717        }
2718
2719        let commit_guard = RemoteTuplePublicationCommitGuard {
2720            guards,
2721            backend_lease,
2722            revalidated_tiered_source,
2723        };
2724        if commit_guard.is_lock_lost() {
2725            return Err(Error::other("data movement publication fence was lost after source revalidation"));
2726        }
2727        Ok(commit_guard)
2728    }
2729}
2730
2731impl ECStore {
2732    /// Acquire the pre-copy half of a data-movement publication fence. Legacy
2733    /// remote objects without a destination identity pin the named tier; an
2734    /// ordinary local source takes the zero-tier-manager-work `None` lease
2735    /// branch but still receives source-at-commit validation.
2736    pub(crate) async fn acquire_remote_tuple_publication_fence(
2737        self: &Arc<Self>,
2738        bucket: &str,
2739        source_pool_idx: usize,
2740        source: &ObjectInfo,
2741        include_fixed_domain: bool,
2742    ) -> Result<RemoteTuplePublicationFence> {
2743        if self.pools.get(source_pool_idx).is_none() {
2744            return Err(Error::other_with_context(
2745                "data movement source pool is out of range",
2746                format!("pool index {source_pool_idx} for {bucket}/{}", source.name),
2747            ));
2748        }
2749        let backend_target = remote_tuple_publication_target(source).map_err(Error::other)?;
2750        let include_fixed_domain = include_fixed_domain && backend_target.is_some();
2751        Ok(RemoteTuplePublicationFence {
2752            store: Arc::clone(self),
2753            bucket: bucket.to_string(),
2754            object: encode_dir_object(&source.name),
2755            source_pool_idx,
2756            source: RemoteTuplePublicationSource::Object(source.clone()),
2757            include_fixed_domain,
2758            fixed_read_anchor: None,
2759            backend_target,
2760        })
2761    }
2762
2763    async fn acquire_tiered_remote_tuple_publication_fence(
2764        self: &Arc<Self>,
2765        bucket: &str,
2766        object: &str,
2767        source_pool_idx: usize,
2768        source: &rustfs_filemeta::FileInfo,
2769    ) -> Result<RemoteTuplePublicationFence> {
2770        let object_info = ObjectInfo::from_file_info(source, bucket, object, true);
2771        let Some((tier_name, backend_identity)) = remote_tuple_publication_target(&object_info).map_err(Error::other)? else {
2772            return Err(Error::other("tiered data movement source has no publishable remote tuple"));
2773        };
2774        Ok(RemoteTuplePublicationFence {
2775            store: Arc::clone(self),
2776            bucket: bucket.to_string(),
2777            object: encode_dir_object(object),
2778            source_pool_idx,
2779            source: RemoteTuplePublicationSource::Tiered(source.clone()),
2780            include_fixed_domain: true,
2781            fixed_read_anchor: None,
2782            backend_target: Some((tier_name, backend_identity)),
2783        })
2784    }
2785
2786    pub(crate) fn is_equivalent_decommission_capacity_target(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2787        source.bucket == target.bucket
2788            && source.name == decode_dir_object(&target.name)
2789            && if source.delete_marker {
2790                is_equivalent_data_movement_delete_marker(source, target)
2791            } else {
2792                crate::data_movement::is_equivalent_data_movement_object_identity(source, target, true, false)
2793            }
2794    }
2795
2796    /// Captures Object Lock state once for a batch of PUTs to the same bucket.
2797    /// `handle_put_object` only reuses the token for the same store, bucket,
2798    /// bucket incarnation, and Object Lock configuration revision.
2799    pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
2800        check_valid_bucket_name(bucket)?;
2801        let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
2802        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2803        let (state, bucket_incarnation_id, config_revision) =
2804            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2805        if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
2806            return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
2807        }
2808        Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket(
2809            self.id,
2810            bucket,
2811            bucket_incarnation_id,
2812            config_revision,
2813            state,
2814            lifecycle_guard,
2815            metadata_guard,
2816        )))
2817    }
2818
2819    pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
2820        &self,
2821        bucket: &str,
2822        lifecycle_fence: &NamespaceLockFence,
2823    ) -> Result<Arc<ObjectLockConfigSnapshot>> {
2824        if lifecycle_fence.is_lock_lost() {
2825            return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
2826        }
2827        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2828        let (state, bucket_incarnation_id, config_revision) =
2829            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2830        if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
2831            return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
2832        }
2833        Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
2834            self.id,
2835            bucket,
2836            bucket_incarnation_id,
2837            config_revision,
2838            state,
2839            lifecycle_fence.clone(),
2840            Arc::new(metadata_guard),
2841        )))
2842    }
2843
2844    /// Resolves a GET's object identity without constructing its body reader.
2845    ///
2846    /// This is an additive two-stage counterpart to `get_object_reader`. The
2847    /// existing method remains the compatibility path for callers that do not
2848    /// need a pre-reader decision point.
2849    pub async fn prepare_select_object_snapshot(
2850        &self,
2851        bucket: &str,
2852        object: &str,
2853        headers: &HeaderMap,
2854        opts: &ObjectOptions,
2855    ) -> std::result::Result<SelectObjectSnapshot, PrepareSelectObjectSnapshotError> {
2856        check_get_obj_args(bucket, object)?;
2857
2858        let object = encode_dir_object(object);
2859        let mut opts = opts.clone();
2860        opts.no_lock = false;
2861        opts.metadata_cache_safe = false;
2862        let read_lock_guards = self
2863            .acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
2864            .await?;
2865        if self.ctx.lock_manager().is_disabled() {
2866            return Err(SnapshotConsistencyError::LockingDisabled.into());
2867        }
2868        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2869            return Err(SnapshotConsistencyError::LockLost.into());
2870        }
2871
2872        let (mut metadata, pool) = if self.single_pool() {
2873            let pool = Arc::clone(&self.pools[0]);
2874            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2875            (metadata, pool)
2876        } else {
2877            // Keep the large multi-pool selection future off the caller stack.
2878            // Debug builds otherwise exceed the common 2 MiB worker stack.
2879            Box::pin(async {
2880                let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2881                let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
2882                    StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
2883                })?;
2884                Ok::<_, StorageError>((metadata, pool))
2885            })
2886            .await?
2887        };
2888        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2889            return Err(SnapshotConsistencyError::LockLost.into());
2890        }
2891        if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2892            return Err(error.into());
2893        }
2894
2895        let logical_size_i64 = metadata.object_info().get_actual_size().map_err(StorageError::from)?;
2896        let logical_size = u64::try_from(logical_size_i64)
2897            .map_err(|_| PrepareSelectObjectSnapshotError::InvalidLogicalSize { size: logical_size_i64 })?;
2898        let read_semantics_identity = metadata.read_semantics_identity();
2899        let object_info = metadata.take_object_info();
2900        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2901            return Err(SnapshotConsistencyError::LockLost.into());
2902        }
2903
2904        Ok(SelectObjectSnapshot {
2905            pool,
2906            bucket: bucket.to_owned(),
2907            object,
2908            headers: rustfs_utils::http::project_ssec_transport_headers(headers),
2909            opts,
2910            object_info,
2911            logical_size,
2912            read_semantics_identity,
2913            first_metadata: ParkingMutex::new(Some(metadata)),
2914            lease: Arc::new(SelectObjectSnapshotLease::new(read_lock_guards)),
2915        })
2916    }
2917
2918    pub async fn prepare_get_object_reader(
2919        &self,
2920        bucket: &str,
2921        object: &str,
2922        range: Option<HTTPRangeSpec>,
2923        headers: HeaderMap,
2924        opts: &ObjectOptions,
2925    ) -> Result<PreparedGetObjectReader> {
2926        check_get_obj_args(bucket, object)?;
2927
2928        let object = encode_dir_object(object);
2929        let mut opts = opts.clone();
2930        let read_lock_guard = self
2931            .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
2932            .await?;
2933
2934        let (metadata, pool) = if self.single_pool() {
2935            let pool = Arc::clone(&self.pools[0]);
2936            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2937            (metadata, pool)
2938        } else {
2939            // Keep the large multi-pool selection future off the caller stack.
2940            // Debug builds otherwise exceed the common 2 MiB worker stack.
2941            Box::pin(async {
2942                let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2943                if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2944                    return Err(error);
2945                }
2946                let pool = self
2947                    .pools
2948                    .get(pool_idx)
2949                    .cloned()
2950                    .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
2951                Ok((metadata, pool))
2952            })
2953            .await?
2954        };
2955
2956        Ok(PreparedGetObjectReader {
2957            pool,
2958            bucket: bucket.to_owned(),
2959            object,
2960            range,
2961            headers,
2962            opts,
2963            metadata,
2964            read_lock_guard,
2965        })
2966    }
2967
2968    fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
2969        match err {
2970            rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
2971                mode,
2972                bucket: bucket.to_string(),
2973                object: object.to_string(),
2974                required,
2975                achieved,
2976            },
2977            other => StorageError::Lock(other),
2978        }
2979    }
2980
2981    pub(super) async fn acquire_object_write_lock(
2982        &self,
2983        op: &'static str,
2984        bucket: &str,
2985        object: &str,
2986    ) -> Result<ObjectLockDiagGuard> {
2987        let diag_enabled = is_object_lock_diag_enabled();
2988        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
2989        let acquire_start = Instant::now();
2990        #[cfg(any(test, feature = "test-util"))]
2991        if matches!(op, "delete_object" | "delete_objects") {
2992            notify_delete_namespace_pending(bucket);
2993        }
2994        let timeout = get_lock_acquire_timeout();
2995        let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", timeout);
2996        let result = ns_lock.get_write_lock(timeout).await;
2997        attempt.observe(&result);
2998        let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
2999        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3000        log_object_lock_acquire_if_slow(
3001            op,
3002            bucket,
3003            object,
3004            owner.as_deref(),
3005            ObjectLockDiagMode::Write,
3006            acquire_start.elapsed(),
3007            diag_enabled,
3008        );
3009
3010        Ok(ObjectLockDiagGuard::new(
3011            guard,
3012            diag_enabled,
3013            op,
3014            diag_enabled.then(|| bucket.to_string()),
3015            diag_enabled.then(|| object.to_string()),
3016            owner,
3017            ObjectLockDiagMode::Write,
3018        )
3019        .with_attempt(attempt))
3020    }
3021
3022    async fn acquire_object_write_lock_if_needed(
3023        &self,
3024        op: &'static str,
3025        bucket: &str,
3026        object: &str,
3027        opts: &mut ObjectOptions,
3028    ) -> Result<Option<ObjectLockDiagGuard>> {
3029        if opts.no_lock {
3030            return Ok(None);
3031        }
3032
3033        let guard = self.acquire_object_write_lock(op, bucket, object).await?;
3034        if let Some(signal) = guard.lock_lost_signal() {
3035            opts.add_namespace_lock_lost_signal(signal);
3036        }
3037        opts.ensure_namespace_lock_fence();
3038        opts.no_lock = true;
3039
3040        Ok(Some(guard))
3041    }
3042
3043    /// Serializes the RestoreObject accept path — the read of the current
3044    /// `x-amz-restore` status, the ongoing/already-restored decision, and the
3045    /// metadata write that flips `ongoing-request="true"` — against concurrent
3046    /// accepts of the same object, making that read-check-write an atomic
3047    /// compare-and-set (backlog#1304). While the guard is held the caller must
3048    /// pass `no_lock: true` on its reads/writes of this object, check
3049    /// [`RestoreAcceptGuard::is_lock_lost`] before the status write, and drop
3050    /// the guard before starting the tier copy-back so concurrent
3051    /// HEAD/`get_object_info` stay non-blocking during the restore.
3052    pub async fn acquire_restore_accept_guard(&self, bucket: &str, object: &str) -> Result<RestoreAcceptGuard> {
3053        let object = encode_dir_object(object);
3054        let guard = self
3055            .acquire_object_write_lock("restore_object_accept", bucket, &object)
3056            .await?;
3057        Ok(RestoreAcceptGuard(guard))
3058    }
3059
3060    /// Acquire the liveness lock for a newly generated restore UUID using the
3061    /// normal namespace-lock timeout. New generations cannot legitimately
3062    /// contend, but a distributed quorum can take longer than the short orphan
3063    /// probe window under load.
3064    pub async fn acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<RestoreWorkerGuard> {
3065        let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3066        let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3067        lock.get_write_lock_quiet(get_lock_acquire_timeout())
3068            .await
3069            .map(RestoreWorkerGuard)
3070            .map_err(|err| Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err))
3071    }
3072
3073    /// Probe the liveness lock for an already-persisted restore UUID.
3074    /// Contention is returned as `None`; infrastructure/quorum failures remain
3075    /// typed errors so callers fail closed instead of reaping an active worker.
3076    pub async fn try_acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<Option<RestoreWorkerGuard>> {
3077        let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3078        let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3079        match lock.get_write_lock_quiet(RESTORE_WORKER_LOCK_PROBE_TIMEOUT).await {
3080            Ok(guard) => Ok(Some(RestoreWorkerGuard(guard))),
3081            Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => Ok(None),
3082            Err(err) => Err(Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err)),
3083        }
3084    }
3085
3086    async fn acquire_delete_objects_write_locks(
3087        &self,
3088        bucket: &str,
3089        objects: &[ObjectToDelete],
3090        opts: &mut ObjectOptions,
3091    ) -> Result<Vec<ObjectLockDiagGuard>> {
3092        if opts.no_lock || objects.is_empty() {
3093            return Ok(Vec::new());
3094        }
3095
3096        let object_names = sorted_unique_delete_object_names(objects);
3097        // Lock order: encoded object names are acquired in ascending order, then
3098        // the set-layer calls receive no_lock so they do not reacquire them.
3099        let mut guards = Vec::with_capacity(object_names.len());
3100        for object in object_names {
3101            guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
3102        }
3103        opts.no_lock = true;
3104        for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
3105            opts.add_namespace_lock_lost_signal(signal);
3106        }
3107        opts.ensure_namespace_lock_fence();
3108
3109        Ok(guards)
3110    }
3111
3112    async fn acquire_object_read_lock_if_needed(
3113        &self,
3114        op: &'static str,
3115        bucket: &str,
3116        object: &str,
3117        opts: &mut ObjectOptions,
3118    ) -> Result<Option<ObjectLockDiagGuard>> {
3119        if opts.no_lock {
3120            return Ok(None);
3121        }
3122
3123        let diag_enabled = is_object_lock_diag_enabled();
3124        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
3125        let acquire_start = Instant::now();
3126        let timeout = get_lock_acquire_timeout();
3127        let mut attempt = ObjectLockAttempt::start(op, bucket, object, opts.version_id.as_deref(), &ns_lock, "read", timeout);
3128        let result = ns_lock.get_read_lock(timeout).await;
3129        attempt.observe(&result);
3130        let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3131        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3132        log_object_lock_acquire_if_slow(
3133            op,
3134            bucket,
3135            object,
3136            owner.as_deref(),
3137            ObjectLockDiagMode::Read,
3138            acquire_start.elapsed(),
3139            diag_enabled,
3140        );
3141        opts.no_lock = true;
3142        opts.metadata_cache_safe = true;
3143
3144        Ok(Some(
3145            ObjectLockDiagGuard::new(
3146                guard,
3147                diag_enabled,
3148                op,
3149                diag_enabled.then(|| bucket.to_string()),
3150                diag_enabled.then(|| object.to_string()),
3151                owner,
3152                ObjectLockDiagMode::Read,
3153            )
3154            .with_attempt(attempt),
3155        ))
3156    }
3157
3158    /// Publish a native scanner replica without allowing stale pool selection
3159    /// to race retirement. Failed/canceled membership repair remains permitted.
3160    /// A canceled waiter cannot release publication fences from an in-flight write.
3161    pub async fn save_scanner_pause_backlog_replica(
3162        self: &Arc<Self>,
3163        pool_index: usize,
3164        set_index: usize,
3165        data: Vec<u8>,
3166        preconditions: crate::storage_api_contracts::object::HTTPPreconditions,
3167    ) -> Result<ObjectInfo> {
3168        let set = self
3169            .pools
3170            .get(pool_index)
3171            .and_then(|pool| pool.disk_set.get(set_index))
3172            .ok_or_else(|| Error::InvalidArgument("scanner-backlog".into(), "replica".into(), "unknown pool or set".into()))?;
3173        let set = Arc::clone(set);
3174        let store = Arc::clone(self);
3175        let write = async move {
3176            let object = "buckets/.scanner-pause-backlog.json";
3177            let mut opts = ObjectOptions::default();
3178            // Match migration: fixed object namespace -> durable pool metadata ->
3179            // actual replica namespace. The replica need not be the hash-routed set.
3180            let object_guard = if store.single_pool() {
3181                None
3182            } else {
3183                Some(
3184                    store
3185                        .acquire_object_write_lock("scanner_backlog_replica", RUSTFS_META_BUCKET, object)
3186                        .await?,
3187                )
3188            };
3189            let capacity_guard = if let Some(guard) = object_guard.as_ref() {
3190                guard.add_namespace_lock_fence(&mut opts);
3191                opts.no_lock = match store.pools.first().and_then(|pool| pool.disk_set.first()) {
3192                    Some(fixed) => fixed.shares_namespace_lock_domain(&set).await,
3193                    None => false,
3194                };
3195                let capacity_guard = store
3196                    .acquire_external_decommission_capacity_fence(&[pool_index], DecommissionCapacityAdmission::ScannerBacklog)
3197                    .await?;
3198                opts.add_namespace_lock_guard(&capacity_guard);
3199                Some(capacity_guard)
3200            } else {
3201                None
3202            };
3203            let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
3204                set,
3205                data,
3206                preconditions,
3207                opts,
3208                "publish",
3209            )
3210            .await;
3211            drop(capacity_guard);
3212            drop(object_guard);
3213            result
3214        };
3215        // The set layer may detach its rename owner, even for full-tail writes.
3216        // Keep these outer guards alive until that owner finishes if scanner exits.
3217        tokio::spawn(write).await.map_err(Error::from)?
3218    }
3219
3220    pub(super) async fn run_external_decommission_capacity_object_mutation<T, F, Fut>(
3221        &self,
3222        target_pool_idx: usize,
3223        bucket: &str,
3224        lock_object: &str,
3225        target_object: &str,
3226        opts: ObjectOptions,
3227        operation: F,
3228    ) -> Result<T>
3229    where
3230        F: FnOnce(ObjectOptions) -> Fut,
3231        Fut: std::future::Future<Output = Result<T>>,
3232    {
3233        self.run_external_decommission_capacity_object_operation(
3234            target_pool_idx,
3235            bucket,
3236            (lock_object, target_object),
3237            opts,
3238            false,
3239            operation,
3240        )
3241        .await
3242    }
3243
3244    pub(super) async fn run_external_decommission_capacity_object_delete<T, F, Fut>(
3245        &self,
3246        target_pool_idx: usize,
3247        bucket: &str,
3248        lock_object: &str,
3249        target_object: &str,
3250        opts: ObjectOptions,
3251        operation: F,
3252    ) -> Result<T>
3253    where
3254        F: FnOnce(ObjectOptions) -> Fut,
3255        Fut: std::future::Future<Output = Result<T>>,
3256    {
3257        let capacity_releasing = delete_only_releases_capacity(&opts);
3258        self.run_external_decommission_capacity_object_operation(
3259            target_pool_idx,
3260            bucket,
3261            (lock_object, target_object),
3262            opts,
3263            capacity_releasing,
3264            operation,
3265        )
3266        .await
3267    }
3268
3269    async fn run_external_decommission_capacity_object_operation<T, F, Fut>(
3270        &self,
3271        target_pool_idx: usize,
3272        bucket: &str,
3273        objects: (&str, &str),
3274        mut opts: ObjectOptions,
3275        capacity_releasing: bool,
3276        operation: F,
3277    ) -> Result<T>
3278    where
3279        F: FnOnce(ObjectOptions) -> Fut,
3280        Fut: std::future::Future<Output = Result<T>>,
3281    {
3282        let (lock_object, target_object) = objects;
3283        if self.single_pool() {
3284            return operation(opts).await;
3285        }
3286        let (capacity_guard, has_active_decommission) = if capacity_releasing {
3287            self.acquire_decommission_capacity_release_fence_with_active_source().await?
3288        } else {
3289            self.acquire_external_decommission_capacity_fence_with_active_source(
3290                &[target_pool_idx],
3291                DecommissionCapacityAdmission::Mutation,
3292            )
3293            .await?
3294        };
3295        let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3296            // Active migration acquires the object namespace before its capacity
3297            // write. Match that order, then recheck capacity admission.
3298            drop(capacity_guard);
3299            #[cfg(test)]
3300            crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3301            let guard = self
3302                .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3303                .await?;
3304            self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
3305                .await;
3306            let capacity_guard = if capacity_releasing {
3307                self.acquire_decommission_capacity_release_fence_with_active_source().await?.0
3308            } else {
3309                self.acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Mutation)
3310                    .await?
3311            };
3312            (capacity_guard, Some(guard))
3313        } else {
3314            (capacity_guard, None)
3315        };
3316
3317        let result = operation(opts).await;
3318        drop(capacity_guard);
3319        drop(object_guard);
3320        result
3321    }
3322
3323    /// Finish an external object mutation's staged phase by acquiring the
3324    /// fixed namespace before the target-side commit lock. The caller must
3325    /// recheck capacity only after all applicable namespaces are held.
3326    ///
3327    /// Callers must invoke this only after consuming and staging their input
3328    /// stream. The returned namespace guard remains owned by the caller until
3329    /// its local commit completes; `target_lock_covered` tells the set layer
3330    /// whether that guard also covers its target namespace. When no
3331    /// decommission is active, the returned capacity guard is the admission
3332    /// probe and must likewise remain held until the commit completes.
3333    pub(crate) async fn acquire_external_decommission_commit_guards(
3334        &self,
3335        target_pool_idx: usize,
3336        bucket: &str,
3337        object: &str,
3338        no_lock: bool,
3339        admission: DecommissionCapacityAdmission,
3340    ) -> Result<(Option<ObjectLockDiagGuard>, bool, Option<rustfs_lock::NamespaceLockGuard>)> {
3341        let (capacity_guard, has_active_decommission) = self
3342            .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], admission)
3343            .await?;
3344        if !has_active_decommission {
3345            // Keep the read probe through the staged commit. This closes the
3346            // activation gap between admission and publication.
3347            #[cfg(test)]
3348            {
3349                crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id);
3350                crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await;
3351            }
3352            return Ok((None, false, Some(capacity_guard)));
3353        }
3354        drop(capacity_guard);
3355        if no_lock {
3356            return Ok((None, false, None));
3357        }
3358
3359        // Active migration holds the fixed object namespace before taking its
3360        // capacity write fence. Drop the probe and acquire that namespace
3361        // before rechecking capacity, so the staged external commit cannot
3362        // form capacity-read -> object-write against the migration path.
3363        let object_guard = self
3364            .acquire_object_write_lock("external_capacity_order", bucket, object)
3365            .await?;
3366        let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3367        let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3368        let target_lock_covered = match (fixed_set, target_set) {
3369            (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3370            _ => false,
3371        };
3372        Ok((Some(object_guard), target_lock_covered, None))
3373    }
3374
3375    pub(super) async fn run_external_decommission_capacity_heal<T, F, Fut>(
3376        &self,
3377        target_pool_idx: usize,
3378        bucket: &str,
3379        lock_object: &str,
3380        mut opts: HealOpts,
3381        operation: F,
3382    ) -> Result<T>
3383    where
3384        F: FnOnce(HealOpts) -> Fut,
3385        Fut: std::future::Future<Output = Result<T>>,
3386    {
3387        if self.single_pool() {
3388            return operation(opts).await;
3389        }
3390        let (capacity_guard, has_active_decommission) = self
3391            .acquire_external_decommission_capacity_fence_with_active_source(
3392                &[target_pool_idx],
3393                DecommissionCapacityAdmission::Heal,
3394            )
3395            .await?;
3396        let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3397            // Active migration acquires the object namespace before its capacity
3398            // write. Match that order, then recheck capacity admission.
3399            drop(capacity_guard);
3400            #[cfg(test)]
3401            crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3402            let guard = self
3403                .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3404                .await?;
3405            let target_set = self
3406                .pools
3407                .get(target_pool_idx)
3408                .ok_or_else(|| Error::other("heal target pool is unavailable"))?
3409                .get_disks_for_heal_object(lock_object, &opts)?;
3410            let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) {
3411                Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await,
3412                None => false,
3413            };
3414            let capacity_guard = self
3415                .acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Heal)
3416                .await?;
3417            opts.no_lock = target_lock_covered;
3418            (capacity_guard, Some(guard))
3419        } else {
3420            (capacity_guard, None)
3421        };
3422
3423        #[cfg(test)]
3424        if !opts.no_lock {
3425            crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
3426        }
3427        let result = operation(opts).await;
3428        drop(capacity_guard);
3429        drop(object_guard);
3430        result
3431    }
3432
3433    pub(crate) async fn acquire_decommission_object_mutation_fence(
3434        &self,
3435        bucket: &str,
3436        object: &str,
3437    ) -> Result<DecommissionFixedReadAnchor> {
3438        if self.ctx.lock_manager().is_disabled() {
3439            return Err(Error::other("decommission object migration requires namespace locking"));
3440        }
3441
3442        #[cfg(test)]
3443        let test_namespace_lock_fence =
3444            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration);
3445        let object = encode_dir_object(object);
3446        let mut opts = ObjectOptions::default();
3447        let guard = self
3448            .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts)
3449            .await?
3450            .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?;
3451        #[cfg(test)]
3452        let guard = {
3453            let mut guard = guard;
3454            guard.test_namespace_lock_fence = test_namespace_lock_fence;
3455            guard
3456        };
3457        Ok(DecommissionFixedReadAnchor { guard })
3458    }
3459
3460    pub(super) async fn apply_decommission_target_mutation_fence(
3461        &self,
3462        target_pool_idx: usize,
3463        object: &str,
3464        opts: &mut ObjectOptions,
3465        mutation_fence: Option<&ObjectLockDiagGuard>,
3466    ) {
3467        let Some(mutation_fence) = mutation_fence else {
3468            return;
3469        };
3470
3471        mutation_fence.add_namespace_lock_fence(opts);
3472        let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3473        let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3474        opts.no_lock = match (fixed_set, target_set) {
3475            (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3476            _ => false,
3477        };
3478    }
3479
3480    pub(crate) async fn acquire_decommission_source_cleanup_fence(
3481        &self,
3482        bucket: &str,
3483        object: &str,
3484        source_set: &SetDisks,
3485    ) -> Result<SourceCleanupMutationFence> {
3486        if self.ctx.lock_manager().is_disabled() {
3487            return Err(Error::other("decommission source cleanup requires namespace locking"));
3488        }
3489
3490        #[cfg(test)]
3491        crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object);
3492        #[cfg(test)]
3493        let test_namespace_lock_fence =
3494            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup);
3495        let object = encode_dir_object(object);
3496        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3497        let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await;
3498        // Lock order: fixed store mutation domain first; source cleanup takes its
3499        // hashed source-domain lock second only when this guard does not cover it.
3500        let guard = self
3501            .acquire_object_write_lock("decommission_source_cleanup", bucket, &object)
3502            .await?;
3503        #[cfg(test)]
3504        let guard = {
3505            let mut guard = guard;
3506            guard.test_namespace_lock_fence = test_namespace_lock_fence;
3507            guard
3508        };
3509
3510        Ok(SourceCleanupMutationFence {
3511            guard,
3512            source_lock_covered,
3513        })
3514    }
3515
3516    pub(crate) async fn acquire_all_object_read_locks(
3517        &self,
3518        op: &'static str,
3519        bucket: &str,
3520        object: &str,
3521        opts: &mut ObjectOptions,
3522    ) -> Result<Vec<ObjectLockDiagGuard>> {
3523        let diag_enabled = is_object_lock_diag_enabled();
3524        let mut guards = Vec::with_capacity(self.pools.len() + 1);
3525
3526        // Lock order is the store fixed domain first, then pool index ascending
3527        // for each object's hashed set. DELETE and same-key CopyObject use the
3528        // fixed domain, while PUT commits and data movement use the hashed set.
3529        let distributed = self.ctx.is_dist_erasure().await;
3530        if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3531            guards.push(guard);
3532        }
3533        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3534        let mut locked_sets = vec![fixed_set];
3535
3536        for pool in &self.pools {
3537            let hashed_set = pool.get_disks_by_key(object);
3538            let mut lock_domain_already_held = !distributed;
3539            if !lock_domain_already_held {
3540                for locked_set in &locked_sets {
3541                    if locked_set.shares_namespace_lock_domain(&hashed_set).await {
3542                        lock_domain_already_held = true;
3543                        break;
3544                    }
3545                }
3546            }
3547            if lock_domain_already_held {
3548                continue;
3549            }
3550            let ns_lock = hashed_set.new_ns_lock(bucket, object).await?;
3551            let acquire_start = Instant::now();
3552            let guard = ns_lock
3553                .get_read_lock(get_lock_acquire_timeout())
3554                .await
3555                .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3556            let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3557            log_object_lock_acquire_if_slow(
3558                op,
3559                bucket,
3560                object,
3561                owner.as_deref(),
3562                ObjectLockDiagMode::Read,
3563                acquire_start.elapsed(),
3564                diag_enabled,
3565            );
3566            guards.push(ObjectLockDiagGuard::new(
3567                guard,
3568                diag_enabled,
3569                op,
3570                diag_enabled.then(|| bucket.to_string()),
3571                diag_enabled.then(|| object.to_string()),
3572                owner,
3573                ObjectLockDiagMode::Read,
3574            ));
3575            locked_sets.push(hashed_set);
3576        }
3577        Ok(guards)
3578    }
3579
3580    /// Acquire the fixed store domain followed by every physical set domain in
3581    /// stable `(pool_idx, set_idx)` order. This stronger form is reserved for
3582    /// remote-tuple destruction proofs: old topology generations and failed
3583    /// movements may leave a valid source outside the current hash-selected
3584    /// set, so a hashed-only lock/read is not authoritative.
3585    pub(crate) async fn acquire_all_physical_object_read_locks(
3586        &self,
3587        op: &'static str,
3588        bucket: &str,
3589        object: &str,
3590        opts: &mut ObjectOptions,
3591    ) -> Result<Vec<ObjectLockDiagGuard>> {
3592        let diag_enabled = is_object_lock_diag_enabled();
3593        let distributed = self.ctx.is_dist_erasure().await;
3594        let mut guards = Vec::new();
3595        if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3596            guards.push(guard);
3597        }
3598
3599        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3600        let mut locked_sets = vec![fixed_set];
3601        for pool in &self.pools {
3602            for set in &pool.disk_set {
3603                let mut lock_domain_already_held = !distributed;
3604                if !lock_domain_already_held {
3605                    for locked_set in &locked_sets {
3606                        if locked_set.shares_namespace_lock_domain(set).await {
3607                            lock_domain_already_held = true;
3608                            break;
3609                        }
3610                    }
3611                }
3612                if lock_domain_already_held {
3613                    continue;
3614                }
3615                let ns_lock = set.new_ns_lock(bucket, object).await?;
3616                let acquire_start = Instant::now();
3617                let guard = ns_lock
3618                    .get_read_lock(get_lock_acquire_timeout())
3619                    .await
3620                    .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3621                let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3622                log_object_lock_acquire_if_slow(
3623                    op,
3624                    bucket,
3625                    object,
3626                    owner.as_deref(),
3627                    ObjectLockDiagMode::Read,
3628                    acquire_start.elapsed(),
3629                    diag_enabled,
3630                );
3631                guards.push(ObjectLockDiagGuard::new(
3632                    guard,
3633                    diag_enabled,
3634                    op,
3635                    diag_enabled.then(|| bucket.to_string()),
3636                    diag_enabled.then(|| object.to_string()),
3637                    owner,
3638                    ObjectLockDiagMode::Read,
3639                ));
3640                locked_sets.push(Arc::clone(set));
3641            }
3642        }
3643        Ok(guards)
3644    }
3645
3646    pub(crate) async fn acquire_all_physical_object_write_locks(
3647        &self,
3648        op: &'static str,
3649        bucket: &str,
3650        object: &str,
3651    ) -> Result<Vec<ObjectLockDiagGuard>> {
3652        self.acquire_physical_object_write_locks(op, bucket, object, false).await
3653    }
3654
3655    async fn acquire_remaining_physical_object_write_locks(
3656        &self,
3657        op: &'static str,
3658        bucket: &str,
3659        object: &str,
3660    ) -> Result<Vec<ObjectLockDiagGuard>> {
3661        self.acquire_physical_object_write_locks(op, bucket, object, true).await
3662    }
3663
3664    async fn acquire_physical_object_write_locks(
3665        &self,
3666        op: &'static str,
3667        bucket: &str,
3668        object: &str,
3669        store_lock_already_held: bool,
3670    ) -> Result<Vec<ObjectLockDiagGuard>> {
3671        // Caller-held bucket lifecycle and tier-generation guards precede the
3672        // fixed store domain, followed by every distinct physical set domain
3673        // in stable `(pool_idx, set_idx)` order. Exact-object predecessor
3674        // replay sets `store_lock_already_held` only while retaining that same
3675        // fixed-domain write lock.
3676        if self.ctx.lock_manager().is_disabled() {
3677            return Err(Error::other("physical object mutation requires namespace locking"));
3678        }
3679        let diag_enabled = is_object_lock_diag_enabled();
3680        let distributed = self.ctx.is_dist_erasure().await;
3681        let mut guards = Vec::new();
3682        if !store_lock_already_held {
3683            guards.push(self.acquire_object_write_lock(op, bucket, object).await?);
3684        }
3685        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3686        let mut locked_sets = vec![fixed_set];
3687        for pool in &self.pools {
3688            for set in &pool.disk_set {
3689                let mut lock_domain_already_held = !distributed;
3690                if !lock_domain_already_held {
3691                    for locked_set in &locked_sets {
3692                        if locked_set.shares_namespace_lock_domain(set).await {
3693                            lock_domain_already_held = true;
3694                            break;
3695                        }
3696                    }
3697                }
3698                if lock_domain_already_held {
3699                    continue;
3700                }
3701                let ns_lock = set.new_ns_lock(bucket, object).await?;
3702                let acquire_start = Instant::now();
3703                let guard = ns_lock
3704                    .get_write_lock(get_lock_acquire_timeout())
3705                    .await
3706                    .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3707                let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3708                log_object_lock_acquire_if_slow(
3709                    op,
3710                    bucket,
3711                    object,
3712                    owner.as_deref(),
3713                    ObjectLockDiagMode::Write,
3714                    acquire_start.elapsed(),
3715                    diag_enabled,
3716                );
3717                guards.push(ObjectLockDiagGuard::new(
3718                    guard,
3719                    diag_enabled,
3720                    op,
3721                    diag_enabled.then(|| bucket.to_string()),
3722                    diag_enabled.then(|| object.to_string()),
3723                    owner,
3724                    ObjectLockDiagMode::Write,
3725                ));
3726                locked_sets.push(Arc::clone(set));
3727            }
3728        }
3729        Ok(guards)
3730    }
3731
3732    pub(crate) async fn acquire_data_movement_publication_write_locks(
3733        &self,
3734        bucket: &str,
3735        object: &str,
3736        source_pool_idx: usize,
3737        target_pool_idx: usize,
3738        include_fixed_domain: bool,
3739    ) -> Result<Vec<ObjectLockDiagGuard>> {
3740        if self.ctx.lock_manager().is_disabled() {
3741            return Err(Error::other("data movement publication requires namespace locking"));
3742        }
3743        let distributed = self.ctx.is_dist_erasure().await;
3744        let diag_enabled = is_object_lock_diag_enabled();
3745        let mut pool_indices = [source_pool_idx, target_pool_idx];
3746        pool_indices.sort_unstable();
3747        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3748        let mut locked_sets = Vec::with_capacity(3);
3749        let mut guards = Vec::with_capacity(3);
3750        #[cfg(test)]
3751        let test_namespace_lock_fence = decommission_mutation_fence_for_test(
3752            bucket,
3753            &decode_dir_object(object),
3754            DecommissionMutationFenceTestPhase::Migration,
3755        );
3756
3757        // Decommission publishers take the fixed store domain first to match
3758        // tier-delete recovery. Rebalance publishers do not need that global
3759        // serialization, but both paths take source/target domains in stable
3760        // pool order and revalidate the source before target publication.
3761        if include_fixed_domain {
3762            let guard = self
3763                .acquire_object_write_lock("data_movement_publication", bucket, object)
3764                .await?;
3765            #[cfg(test)]
3766            let guard = {
3767                let mut guard = guard;
3768                guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3769                guard
3770            };
3771            guards.push(guard);
3772            locked_sets.push(fixed_set);
3773        }
3774        for pool_idx in pool_indices {
3775            let pool = self
3776                .pools
3777                .get(pool_idx)
3778                .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
3779            let set = pool.get_disks_by_key(object);
3780            let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
3781            if !lock_domain_already_held {
3782                for locked_set in &locked_sets {
3783                    if locked_set.shares_namespace_lock_domain(&set).await {
3784                        lock_domain_already_held = true;
3785                        break;
3786                    }
3787                }
3788            }
3789            if lock_domain_already_held {
3790                continue;
3791            }
3792            let ns_lock = set.new_ns_lock(bucket, object).await?;
3793            let acquire_start = Instant::now();
3794            let guard = ns_lock
3795                .get_write_lock(get_lock_acquire_timeout())
3796                .await
3797                .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3798            let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3799            log_object_lock_acquire_if_slow(
3800                "data_movement_publication",
3801                bucket,
3802                object,
3803                owner.as_deref(),
3804                ObjectLockDiagMode::Write,
3805                acquire_start.elapsed(),
3806                diag_enabled,
3807            );
3808            let guard = ObjectLockDiagGuard::new(
3809                guard,
3810                diag_enabled,
3811                "data_movement_publication",
3812                diag_enabled.then(|| bucket.to_string()),
3813                diag_enabled.then(|| object.to_string()),
3814                owner,
3815                ObjectLockDiagMode::Write,
3816            );
3817            #[cfg(test)]
3818            let guard = {
3819                let mut guard = guard;
3820                if guards.is_empty() {
3821                    guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3822                }
3823                guard
3824            };
3825            guards.push(guard);
3826            locked_sets.push(set);
3827        }
3828        Ok(guards)
3829    }
3830
3831    fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
3832        if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
3833            return reader;
3834        }
3835
3836        if let Some(guard) = guard {
3837            reader.stream = Box::new(LockGuardedReader {
3838                inner: reader.stream,
3839                guard: Some(guard),
3840            });
3841        }
3842
3843        reader
3844    }
3845
3846    async fn get_latest_accessible_object_info_with_idx(
3847        &self,
3848        bucket: &str,
3849        object: &str,
3850        opts: &ObjectOptions,
3851    ) -> Result<(ObjectInfo, usize)> {
3852        let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?;
3853        resolve_latest_object_access(bucket, object, info, idx, opts)
3854    }
3855
3856    pub(crate) async fn select_data_movement_pool_idx(
3857        &self,
3858        bucket: &str,
3859        object: &str,
3860        size: i64,
3861        opts: &ObjectOptions,
3862        no_lock: bool,
3863    ) -> Result<usize> {
3864        let capacity_owner = DecommissionCapacityOwner::from_options(opts);
3865        match self
3866            .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
3867            .await
3868        {
3869            Ok((pinfo, _)) => {
3870                if let Some(owner) = capacity_owner {
3871                    if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
3872                        return Ok(pinfo.index);
3873                    }
3874                } else {
3875                    return Ok(pinfo.index);
3876                }
3877            }
3878            Err(err) => {
3879                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
3880                    return Err(err);
3881                }
3882            }
3883        }
3884        if let Some(owner) = capacity_owner {
3885            let expected_data_bytes = opts
3886                .capacity_expected_data_bytes()
3887                .or_else(|| usize::try_from(size).ok())
3888                .unwrap_or_default();
3889            return self
3890                .select_decommission_capacity_target_pool(owner, expected_data_bytes)
3891                .await;
3892        }
3893
3894        self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
3895    }
3896
3897    async fn find_data_movement_target_info(
3898        &self,
3899        bucket: &str,
3900        object: &str,
3901        target_pool_idx: usize,
3902        opts: &ObjectOptions,
3903    ) -> Result<Option<ObjectInfo>> {
3904        let mut lookup_opts = version_aware_lookup_opts(opts, true);
3905        lookup_opts.include_part_checksums = true;
3906
3907        let Some(pool) = self.pools.get(target_pool_idx) else {
3908            return Err(Error::other(format!(
3909                "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
3910            )));
3911        };
3912
3913        match pool.get_object_info(bucket, object, &lookup_opts).await {
3914            Ok(info) => Ok(Some(info)),
3915            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
3916            Err(err) => Err(err),
3917        }
3918    }
3919
3920    async fn has_equivalent_data_movement_delete_marker(
3921        &self,
3922        bucket: &str,
3923        object: &str,
3924        source: &ObjectInfo,
3925        opts: &ObjectOptions,
3926        target_pool_idx: usize,
3927    ) -> Result<bool> {
3928        let equivalent = resolve_data_movement_delete_marker_resume_result(
3929            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3930                .await,
3931            source,
3932            opts.src_pool_idx,
3933            target_pool_idx,
3934        )?;
3935        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3936            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3937                .await?;
3938        }
3939        Ok(equivalent)
3940    }
3941
3942    async fn has_equivalent_data_movement_tiered_object(
3943        &self,
3944        bucket: &str,
3945        object: &str,
3946        source: &rustfs_filemeta::FileInfo,
3947        opts: &ObjectOptions,
3948        target_pool_idx: usize,
3949    ) -> Result<bool> {
3950        let equivalent = resolve_data_movement_tiered_resume_result(
3951            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3952                .await,
3953            source,
3954            opts.src_pool_idx,
3955            target_pool_idx,
3956        )?;
3957        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3958            let expected_data_bytes = usize::try_from(source.size).unwrap_or_default();
3959            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
3960                .await?;
3961        }
3962        Ok(equivalent)
3963    }
3964
3965    async fn has_equivalent_data_movement_tier_free_version(
3966        &self,
3967        bucket: &str,
3968        object: &str,
3969        source: &rustfs_filemeta::FileInfo,
3970        opts: &ObjectOptions,
3971        target_pool_idx: usize,
3972    ) -> Result<bool> {
3973        let pool = self
3974            .pools
3975            .get(target_pool_idx)
3976            .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
3977        let equivalent = pool
3978            .get_disks_by_key(object)
3979            .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
3980            .await?;
3981        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3982            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3983                .await?;
3984        }
3985        Ok(equivalent)
3986    }
3987
3988    fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
3989        result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
3990    }
3991
3992    fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
3993        result.map_err(|err| {
3994            crate::data_movement::data_movement_context_error(
3995                format!("failed to decommission tiered object for {bucket}/{object}: {err}"),
3996                err,
3997            )
3998        })
3999    }
4000
4001    #[instrument(skip(self, fi, opts))]
4002    pub(crate) async fn decommission_tiered_object(
4003        self: &Arc<Self>,
4004        bucket: &str,
4005        object: &str,
4006        fi: &rustfs_filemeta::FileInfo,
4007        opts: &ObjectOptions,
4008    ) -> Result<()> {
4009        check_put_object_args(bucket, object)?;
4010
4011        let mut opts = opts.clone();
4012        let is_free_version = fi.tier_free_version();
4013        if is_free_version {
4014            opts.incl_free_versions = true;
4015        }
4016        let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
4017            None
4018        } else {
4019            let expected = opts
4020                .expected_bucket_incarnation_id
4021                .ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
4022            let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
4023            if let Some(namespace_guard) = guard.namespace_lock_guard() {
4024                opts.add_bucket_lifecycle_lock_guard(namespace_guard);
4025            }
4026            Some(guard)
4027        };
4028
4029        // Capture exact source/tier identity now; the tier lease and namespace
4030        // writes are acquired only at the final metadata publication boundary.
4031        let publication_fence = self
4032            .acquire_tiered_remote_tuple_publication_fence(bucket, object, opts.src_pool_idx, fi)
4033            .await?;
4034
4035        let logical_object = object;
4036        let object = encode_dir_object(logical_object);
4037        ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4038        if self.single_pool() {
4039            return Self::resolve_decommission_tiered_object_result(
4040                Err(Error::other("single pool deployments cannot decommission tiered objects")),
4041                bucket,
4042                &object,
4043            );
4044        }
4045
4046        let idx = if uses_data_movement_pool_selection(&opts) {
4047            Self::resolve_decommission_target_pool_idx_result(
4048                self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
4049                    .await,
4050                bucket,
4051                &object,
4052            )?
4053        } else {
4054            Self::resolve_decommission_target_pool_idx_result(
4055                self.get_pool_idx_no_lock(bucket, &object, fi.size).await,
4056                bucket,
4057                &object,
4058            )?
4059        };
4060        #[cfg(all(test, feature = "test-util"))]
4061        if is_free_version {
4062            pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
4063        }
4064        let publication_guard = publication_fence.into_commit_guard(idx, bucket, &object).await?;
4065        publication_guard.add_namespace_lock_fence(&mut opts);
4066        opts.no_lock = true;
4067        let mut fi = publication_guard
4068            .revalidated_tiered_source()
4069            .ok_or_else(|| Error::other("tiered data movement publication guard is missing its revalidated source"))?
4070            .clone();
4071        if opts.data_movement {
4072            crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
4073        }
4074        if opts.data_movement && idx == opts.src_pool_idx {
4075            let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4076                Some(
4077                    self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default())
4078                        .await?,
4079                )
4080            } else {
4081                self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
4082                    .await
4083            };
4084            let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
4085            if is_free_version && target_pool_idx == opts.src_pool_idx {
4086                return Err(Error::DiskFull);
4087            }
4088            let equivalent = if is_free_version {
4089                self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
4090                    .await?
4091            } else {
4092                self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
4093                    .await?
4094            };
4095            if equivalent {
4096                return Ok(());
4097            }
4098
4099            return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
4100        }
4101
4102        let expected_data_bytes = usize::try_from(fi.size).ok();
4103        let result = self
4104            .run_decommission_capacity_admitted_mutation(
4105                idx,
4106                DecommissionCapacityOwner::from_options(&opts),
4107                expected_data_bytes,
4108                || async {
4109                    if is_free_version {
4110                        self.pools[idx]
4111                            .get_disks_by_key(&object)
4112                            .decommission_tier_free_version(bucket, &object, &fi, &opts)
4113                            .await
4114                    } else {
4115                        self.pools[idx]
4116                            .get_disks_by_key(&object)
4117                            .decommission_tiered_object(bucket, &object, &fi, &opts)
4118                            .await
4119                    }
4120                },
4121            )
4122            .await;
4123        if matches!(result, Err(Error::PreconditionFailed)) {
4124            if self
4125                .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
4126                .await?
4127            {
4128                return Ok(());
4129            }
4130            return Err(StorageError::DataMovementOverwriteErr(
4131                bucket.to_owned(),
4132                object,
4133                opts.version_id.clone().unwrap_or_default(),
4134            ));
4135        }
4136        if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
4137            return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
4138        }
4139        Self::resolve_decommission_tiered_object_result(result, bucket, &object)
4140    }
4141
4142    /// Open a source reader for a server-side copy.
4143    ///
4144    /// Copy consumers hold the source reader while a destination write can
4145    /// apply backpressure. Keep that read contract explicit at the storage
4146    /// boundary so the lower-level legacy multipart pipeline can suppress its
4147    /// speculative next-part setup without changing the public `ObjectIO`
4148    /// trait or `ObjectOptions` layout.
4149    pub async fn get_object_reader_for_copy(
4150        &self,
4151        bucket: &str,
4152        object: &str,
4153        range: Option<HTTPRangeSpec>,
4154        h: HeaderMap,
4155        opts: &ObjectOptions,
4156    ) -> Result<(GetObjectReader, tokio_util::sync::CancellationToken)> {
4157        let cancellation = tokio_util::sync::CancellationToken::new();
4158        let reader = crate::set_disk::with_get_object_read_cancellation(
4159            cancellation.clone(),
4160            crate::set_disk::with_get_object_read_policy(
4161                crate::set_disk::GetObjectReadPolicy::CopySource,
4162                self.handle_get_object_reader(bucket, object, range, h, opts),
4163            ),
4164        )
4165        .await?;
4166        Ok((reader, cancellation))
4167    }
4168
4169    #[instrument(level = "debug", skip(self, h))]
4170    #[hotpath::measure(impl_type = "ECStore")]
4171    pub(super) async fn handle_get_object_reader(
4172        &self,
4173        bucket: &str,
4174        object: &str,
4175        range: Option<HTTPRangeSpec>,
4176        h: HeaderMap,
4177        opts: &ObjectOptions,
4178    ) -> Result<GetObjectReader> {
4179        check_get_obj_args(bucket, object)?;
4180
4181        let object = rustfs_utils::path::encode_dir_object_ref(object);
4182        let mut opts = opts.clone();
4183        let read_lock_guard = self
4184            .acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
4185            .await?;
4186
4187        let reader = if self.single_pool() {
4188            self.pools[0]
4189                .get_object_reader(bucket, object.as_ref(), range, h, &opts)
4190                .await?
4191        } else {
4192            // Keep selection plus prepared-open state off the caller stack.
4193            // Debug builds otherwise exceed the common 2 MiB worker stack.
4194            Box::pin(async {
4195                let (metadata, idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
4196                if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
4197                    return Err(error);
4198                }
4199                self.pools[idx]
4200                    .get_object_reader_with_prepared_metadata(bucket, object.as_ref(), range, h, &opts, metadata)
4201                    .await
4202            })
4203            .await?
4204        };
4205
4206        Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
4207    }
4208
4209    async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
4210        check_put_object_args(bucket, object)?;
4211
4212        let object = encode_dir_object(object);
4213        let mut opts = opts.clone();
4214        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
4215            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
4216        }
4217        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
4218            let expected_incarnation_id = opts
4219                .expected_bucket_incarnation_id
4220                .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
4221            if opts.object_lock_config_snapshot.is_none() {
4222                opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
4223            }
4224            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
4225                Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
4226                    Arc::clone(snapshot)
4227                }
4228                _ => {
4229                    return Err(Error::other(
4230                        "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
4231                    ));
4232                }
4233            };
4234            snapshot.add_lock_fences(&mut opts);
4235        }
4236        Ok((object, opts))
4237    }
4238
4239    async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
4240        if self.single_pool() {
4241            return Ok(0);
4242        }
4243
4244        let idx = if uses_data_movement_pool_selection(opts) {
4245            self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
4246        } else if opts.no_lock {
4247            self.get_pool_idx_no_lock(bucket, object, size).await?
4248        } else {
4249            self.get_pool_idx(bucket, object, size).await?
4250        };
4251
4252        if opts.data_movement && idx == opts.src_pool_idx {
4253            return Err(StorageError::DataMovementOverwriteErr(
4254                bucket.to_owned(),
4255                object.to_owned(),
4256                opts.version_id.clone().unwrap_or_default(),
4257            ));
4258        }
4259        Ok(idx)
4260    }
4261
4262    #[cfg(test)]
4263    pub(crate) async fn put_object_for_data_movement(
4264        self: &Arc<Self>,
4265        bucket: &str,
4266        object: &str,
4267        data: &mut PutObjReader,
4268        opts: &ObjectOptions,
4269        mutation_fence: Option<&ObjectLockDiagGuard>,
4270    ) -> Result<(usize, Result<ObjectInfo>)> {
4271        self.put_object_for_data_movement_inner(bucket, object, data, opts, mutation_fence, None)
4272            .await
4273    }
4274
4275    pub(crate) async fn put_object_for_data_movement_with_publication_fence(
4276        self: &Arc<Self>,
4277        bucket: &str,
4278        object: &str,
4279        data: &mut PutObjReader,
4280        opts: &ObjectOptions,
4281        publication_fence: RemoteTuplePublicationFence,
4282    ) -> Result<(usize, Result<ObjectInfo>)> {
4283        self.put_object_for_data_movement_inner(bucket, object, data, opts, None, Some(publication_fence))
4284            .await
4285    }
4286
4287    async fn put_object_for_data_movement_inner(
4288        self: &Arc<Self>,
4289        bucket: &str,
4290        object: &str,
4291        data: &mut PutObjReader,
4292        opts: &ObjectOptions,
4293        mutation_fence: Option<&ObjectLockDiagGuard>,
4294        publication_fence: Option<RemoteTuplePublicationFence>,
4295    ) -> Result<(usize, Result<ObjectInfo>)> {
4296        if !opts.data_movement {
4297            return Err(Error::other("data movement PUT requires data_movement options"));
4298        }
4299        let request_opts = opts;
4300        let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
4301        ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4302        let idx = self
4303            .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4304            .await?;
4305        let fixed_read_anchor = publication_fence
4306            .as_ref()
4307            .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
4308        self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
4309            .await;
4310        let expected_data_bytes = usize::try_from(data.size()).ok();
4311        let result = self
4312            .run_decommission_capacity_admitted_mutation_with_capacity_lease(
4313                idx,
4314                DecommissionCapacityOwner::from_options(&opts),
4315                expected_data_bytes,
4316                |capacity_lease| async move {
4317                    if let Some(capacity_lease) = capacity_lease {
4318                        opts.add_namespace_lock_lost_signal(capacity_lease);
4319                    }
4320                    let result = match publication_fence {
4321                        Some(publication_fence) => {
4322                            self.pools[idx]
4323                                .put_object_with_old_current_size_for_data_movement(
4324                                    bucket,
4325                                    &object,
4326                                    data,
4327                                    &opts,
4328                                    publication_fence,
4329                                )
4330                                .await
4331                        }
4332                        None => {
4333                            self.pools[idx]
4334                                .put_object_with_old_current_size(bucket, &object, data, &opts)
4335                                .await
4336                        }
4337                    };
4338                    result.map(|(object_info, _)| object_info)
4339                },
4340            )
4341            .await;
4342        let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
4343        if result.is_ok() {
4344            list_objects::observe_list_objects_mutation(self, bucket).await;
4345        }
4346        Ok((idx, result))
4347    }
4348
4349    #[instrument(level = "debug", skip(self, data))]
4350    #[hotpath::measure(impl_type = "ECStore")]
4351    pub(super) async fn handle_put_object(
4352        &self,
4353        bucket: &str,
4354        object: &str,
4355        data: &mut PutObjReader,
4356        opts: &ObjectOptions,
4357    ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
4358        let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
4359        let idx = self
4360            .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4361            .await?;
4362        let mut opts = opts;
4363        if !self.single_pool() {
4364            opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4365        }
4366        let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4367        let result = self.pools[idx]
4368            .put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
4369            .await;
4370        enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4371        result
4372    }
4373
4374    #[instrument(level = "trace", skip(self))]
4375    pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4376        self.get_object_info_snapshot(bucket, object, opts, false).await
4377    }
4378
4379    /// Return metadata for DELETE preflight, including an explicitly addressed
4380    /// delete marker. GET/HEAD may also use this metadata-only lookup to enrich
4381    /// an already failed read with marker headers, never to serve marker data.
4382    /// Normal reads must keep using `get_object_info`; authorization and Object
4383    /// Lock enforcement still belong to the caller and locked delete.
4384    #[instrument(level = "trace", skip_all)]
4385    pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4386        self.get_object_info_snapshot(bucket, object, opts, true).await
4387    }
4388
4389    async fn get_object_info_snapshot(
4390        &self,
4391        bucket: &str,
4392        object: &str,
4393        opts: &ObjectOptions,
4394        allow_delete_marker: bool,
4395    ) -> Result<ObjectInfo> {
4396        check_object_args(bucket, object)?;
4397
4398        let object = encode_dir_object(object);
4399        let mut opts = opts.clone();
4400        let _object_lock_guard = self
4401            .acquire_object_read_lock_if_needed("get_object_info", bucket, &object, &mut opts)
4402            .await?;
4403
4404        let info = if self.single_pool() {
4405            self.pools[0].get_object_info(bucket, object.as_str(), &opts).await?
4406        } else if allow_delete_marker {
4407            self.get_latest_object_info_with_idx(bucket, object.as_str(), &opts).await?.0
4408        } else {
4409            self.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
4410                .await?
4411                .0
4412        };
4413        opts.precondition_check(&info)?;
4414        Ok(info)
4415    }
4416
4417    #[instrument(skip(self))]
4418    #[allow(clippy::too_many_arguments)]
4419    pub(super) async fn handle_copy_object(
4420        &self,
4421        src_bucket: &str,
4422        src_object: &str,
4423        dst_bucket: &str,
4424        dst_object: &str,
4425        src_info: &mut ObjectInfo,
4426        src_opts: &ObjectOptions,
4427        dst_opts: &ObjectOptions,
4428    ) -> Result<ObjectInfo> {
4429        check_copy_obj_args(src_bucket, src_object)?;
4430        check_copy_obj_args(dst_bucket, dst_object)?;
4431
4432        let src_object = encode_dir_object(src_object);
4433        let dst_object = encode_dir_object(dst_object);
4434
4435        let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
4436
4437        let mut dst_opts = dst_opts.clone();
4438        dst_opts.inherit_shard_integrity(src_info);
4439        if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
4440            dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
4441        }
4442        let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
4443            None
4444        } else {
4445            Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
4446        };
4447        let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4448            dst_opts.add_bucket_lifecycle_lock_guard(guard);
4449            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
4450            if dst_opts
4451                .expected_bucket_incarnation_id
4452                .is_some_and(|expected| expected != current_incarnation_id)
4453            {
4454                return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
4455            }
4456            Some(current_incarnation_id)
4457        } else {
4458            dst_opts.expected_bucket_incarnation_id
4459        };
4460        if dst_opts
4461            .bucket_lifecycle_lock_fence
4462            .as_ref()
4463            .is_some_and(NamespaceLockFence::is_lock_lost)
4464        {
4465            return Err(StorageError::NamespaceLockQuorumUnavailable {
4466                mode: "copy_object_bucket_generation",
4467                bucket: dst_bucket.to_string(),
4468                object: dst_object.clone(),
4469                required: 1,
4470                achieved: 0,
4471            });
4472        }
4473        if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) {
4474            let incarnation_id =
4475                current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?;
4476            let lifecycle_fence = dst_opts
4477                .bucket_lifecycle_lock_fence
4478                .as_ref()
4479                .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?;
4480            let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() {
4481                Some(snapshot) => Arc::clone(snapshot),
4482                None => {
4483                    self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence)
4484                        .await?
4485                }
4486            };
4487            if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) {
4488                return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation"));
4489            }
4490            snapshot.add_lock_fences(&mut dst_opts);
4491            dst_opts.object_lock_config_snapshot = Some(snapshot);
4492        }
4493        let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() {
4494            self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts)
4495                .await?
4496        } else {
4497            None
4498        };
4499
4500        if cp_src_dst_same {
4501            let (_, pool_idx) = self
4502                .get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
4503                .await?;
4504            let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
4505
4506            if source_pool_writable
4507                && let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
4508                && src_vid == dst_vid
4509            {
4510                let capacity_object = dst_object.clone();
4511                return self
4512                    .run_external_decommission_capacity_object_mutation(
4513                        pool_idx,
4514                        dst_bucket,
4515                        &capacity_object,
4516                        &capacity_object,
4517                        dst_opts.clone(),
4518                        |opts| async move {
4519                            self.pools[pool_idx]
4520                                .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4521                                .await
4522                        },
4523                    )
4524                    .await;
4525            }
4526
4527            if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
4528                if src_info.metadata_only {
4529                    // Zero-copy update: only xl.meta is rewritten, the data blocks stay as they
4530                    // are. The caller must therefore guarantee that the destination metadata
4531                    // still describes the stored bytes. In particular a copy that re-derives
4532                    // encryption material may NOT set metadata_only — that would leave a fresh
4533                    // DEK beside ciphertext sealed under the old one, permanently destroying the
4534                    // object. The S3 handler enforces this before calling in (see the
4535                    // metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling
4536                    // versioned branch below resolves the same risk by rewriting through
4537                    // put_object (issue #4238).
4538                    let capacity_object = dst_object.clone();
4539                    return self
4540                        .run_external_decommission_capacity_object_mutation(
4541                            pool_idx,
4542                            dst_bucket,
4543                            &capacity_object,
4544                            &capacity_object,
4545                            dst_opts.clone(),
4546                            |opts| async move {
4547                                self.pools[pool_idx]
4548                                    .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4549                                    .await
4550                            },
4551                        )
4552                        .await;
4553                }
4554                // Transitioned object self-copy: restore from tier into the same pool.
4555                let mut put_opts = ObjectOptions {
4556                    user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4557                    versioned: dst_opts.versioned,
4558                    shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4559                    version_id: dst_opts.version_id.clone(),
4560                    no_lock: dst_opts.no_lock,
4561                    mod_time: dst_opts.mod_time,
4562                    http_preconditions: dst_opts.http_preconditions.clone(),
4563                    expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4564                    expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4565                    namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4566                    bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4567                    object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4568                    quota_admission: dst_opts.quota_admission,
4569                    ..Default::default()
4570                };
4571                if !self.single_pool() {
4572                    put_opts.decommission_capacity_admission =
4573                        crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4574                }
4575                return if let Some(reader) = src_info.put_object_reader.as_mut() {
4576                    let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4577                    let result = self.pools[pool_idx]
4578                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
4579                        .await;
4580                    enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4581                    result
4582                } else {
4583                    Err(StorageError::InvalidArgument(
4584                        src_bucket.to_owned(),
4585                        src_object.to_owned(),
4586                        "put_object_reader is none".to_owned(),
4587                    ))
4588                };
4589            }
4590
4591            if source_pool_writable && dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
4592                // Restoring a specific historical version onto the current key creates a NEW
4593                // version. When the caller supplies a reader (S3 CopyObject), write the fetched
4594                // bytes through put_object so any re-encryption/compression applied to the reader
4595                // stays consistent with the new version's metadata. Sharing the source data_dir via
4596                // a metadata-only version copy would corrupt SSE/compressed objects (issue #4238).
4597                if let Some(reader) = src_info.put_object_reader.as_mut() {
4598                    let mut put_opts = ObjectOptions {
4599                        user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4600                        versioned: dst_opts.versioned,
4601                        shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4602                        version_id: dst_opts.version_id.clone(),
4603                        no_lock: dst_opts.no_lock,
4604                        mod_time: dst_opts.mod_time,
4605                        http_preconditions: dst_opts.http_preconditions.clone(),
4606                        expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4607                        expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4608                        namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4609                        bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4610                        object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4611                        quota_admission: dst_opts.quota_admission,
4612                        ..Default::default()
4613                    };
4614                    if !self.single_pool() {
4615                        put_opts.decommission_capacity_admission =
4616                            crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4617                    }
4618                    let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4619                    let result = self.pools[pool_idx]
4620                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
4621                        .await;
4622                    enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4623                    return result;
4624                }
4625                src_info.version_only = true;
4626                let capacity_object = dst_object.clone();
4627                return self
4628                    .run_external_decommission_capacity_object_mutation(
4629                        pool_idx,
4630                        dst_bucket,
4631                        &capacity_object,
4632                        &capacity_object,
4633                        dst_opts.clone(),
4634                        |opts| async move {
4635                            self.pools[pool_idx]
4636                                .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4637                                .await
4638                        },
4639                    )
4640                    .await;
4641            }
4642        }
4643
4644        let pool_idx = if dst_opts.no_lock {
4645            self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
4646        } else {
4647            self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
4648        };
4649        let dst_object_name = dst_object.as_str();
4650
4651        let mut put_opts = ObjectOptions {
4652            user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4653            versioned: dst_opts.versioned,
4654            shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4655            version_id: dst_opts.version_id.clone(),
4656            no_lock: dst_opts.no_lock,
4657            mod_time: dst_opts.mod_time,
4658            http_preconditions: dst_opts.http_preconditions.clone(),
4659            expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4660            expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4661            namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4662            bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4663            object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4664            quota_admission: dst_opts.quota_admission,
4665            ..Default::default()
4666        };
4667        if !self.single_pool() {
4668            put_opts.decommission_capacity_admission =
4669                crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4670        }
4671
4672        if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
4673            let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4674            let result = self.pools[pool_idx]
4675                .put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
4676                .await;
4677            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4678            return result;
4679        }
4680
4681        Err(StorageError::InvalidArgument(
4682            src_bucket.to_owned(),
4683            src_object.to_owned(),
4684            "put_object_reader is none".to_owned(),
4685        ))
4686    }
4687
4688    /// Best-effort purge of an orphan directory prefix — an on-disk tree of empty
4689    /// directories or committed delete residue with no `xl.meta` anywhere. Orphan fragments can sit
4690    /// on any erasure set of any pool (they are left behind by whichever sets stored
4691    /// the now-deleted children), so every set is swept. Returns true when at least
4692    /// one set removed an orphan tree. Hard per-set failures are logged and skipped:
4693    /// the caller falls back to surfacing the original NotFound.
4694    pub(super) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> bool {
4695        let prefix = decode_dir_object(object);
4696        for pool in self.pools.iter() {
4697            for set in pool.disk_set.iter() {
4698                if !set.orphan_purge_has_complete_disk_set().await {
4699                    return false;
4700                }
4701            }
4702        }
4703
4704        let mut purged = false;
4705        for pool in self.pools.iter() {
4706            for set in pool.disk_set.iter() {
4707                match set.purge_orphan_dir_object(bucket, &prefix).await {
4708                    Ok(set_purged) => purged |= set_purged,
4709                    Err(err) => {
4710                        warn!(
4711                            bucket,
4712                            prefix,
4713                            pool_index = pool.pool_idx,
4714                            error = ?err,
4715                            "failed to purge orphan directory prefix"
4716                        );
4717                    }
4718                }
4719            }
4720        }
4721        purged
4722    }
4723
4724    pub(super) async fn purge_orphan_dir_objects_in_bucket(&self, bucket: &str) -> bool {
4725        if is_meta_bucketname(bucket) {
4726            return false;
4727        }
4728        for pool in self.pools.iter() {
4729            for set in pool.disk_set.iter() {
4730                if !set.orphan_purge_has_complete_disk_set().await {
4731                    return false;
4732                }
4733            }
4734        }
4735
4736        let mut purged = false;
4737        for pool in self.pools.iter() {
4738            for set in pool.disk_set.iter() {
4739                purged |= set.purge_orphan_dir_objects_in_bucket(bucket).await;
4740            }
4741        }
4742        purged
4743    }
4744
4745    pub async fn delete_object_with_tier_delete_journal(
4746        self: &Arc<Self>,
4747        bucket: &str,
4748        object: &str,
4749        opts: ObjectOptions,
4750    ) -> Result<ObjectInfo> {
4751        let result = self
4752            .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
4753            .await;
4754        if result.is_ok() {
4755            list_objects::observe_list_objects_mutation(self, bucket).await;
4756        }
4757        result
4758    }
4759
4760    pub async fn delete_objects_with_tier_delete_journal(
4761        self: &Arc<Self>,
4762        bucket: &str,
4763        objects: Vec<ObjectToDelete>,
4764        opts: ObjectOptions,
4765    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
4766        let result = self
4767            .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
4768            .await;
4769        let success_count = result.1.iter().filter(|err| err.is_none()).count();
4770        if success_count > 0 {
4771            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4772        }
4773        result
4774    }
4775
4776    pub async fn delete_objects_with_tier_delete_journal_and_accounting(
4777        self: &Arc<Self>,
4778        bucket: &str,
4779        objects: Vec<ObjectToDelete>,
4780        opts: ObjectOptions,
4781    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
4782        let result = self
4783            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
4784            .await;
4785        let success_count = result.1.iter().filter(|err| err.is_none()).count();
4786        if success_count > 0 {
4787            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4788        }
4789        result
4790    }
4791
4792    #[instrument(skip(self))]
4793    pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
4794        self.handle_delete_object_with_journal(bucket, object, opts, None).await
4795    }
4796
4797    pub(super) async fn handle_delete_object_with_journal(
4798        &self,
4799        bucket: &str,
4800        object: &str,
4801        opts: ObjectOptions,
4802        tier_journal_api: Option<Arc<ECStore>>,
4803    ) -> Result<ObjectInfo> {
4804        Box::pin(async move {
4805            let mut opts = opts;
4806            let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4807            let result = self
4808                .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)
4809                .await;
4810            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4811            result
4812        })
4813        .await
4814    }
4815
4816    async fn handle_delete_object_with_journal_inner(
4817        &self,
4818        bucket: &str,
4819        object: &str,
4820        opts: ObjectOptions,
4821        tier_journal_api: Option<Arc<ECStore>>,
4822    ) -> Result<ObjectInfo> {
4823        check_del_obj_args(bucket, object)?;
4824
4825        if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
4826            return Err(Error::other("lifecycle delete-all requires namespace locking"));
4827        }
4828
4829        let _bucket_lifecycle_guard =
4830            if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
4831                None
4832            } else if opts.delete_prefix {
4833                Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
4834            } else {
4835                Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
4836            };
4837        let object = if opts.delete_prefix && !opts.delete_prefix_object {
4838            object.to_owned()
4839        } else {
4840            encode_dir_object(object)
4841        };
4842        let object = object.as_str();
4843        let mut opts = opts;
4844        ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
4845        let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
4846            Some(get_expiry_configs(self, bucket).await?)
4847        } else {
4848            None
4849        };
4850        opts.tier_delete_journal_api = tier_journal_api.clone();
4851        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4852            opts.add_bucket_lifecycle_lock_guard(guard);
4853        }
4854
4855        if !is_meta_bucketname(bucket) {
4856            get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
4857        }
4858        let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
4859            Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
4860        } else {
4861            None
4862        };
4863        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
4864            opts.add_namespace_lock_guard(guard);
4865        }
4866        let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
4867            let (state, incarnation_id, config_revision) =
4868                get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
4869            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
4870                self.id,
4871                bucket,
4872                incarnation_id,
4873                config_revision,
4874                state,
4875                opts.bucket_lifecycle_lock_fence
4876                    .clone()
4877                    .unwrap_or_else(NamespaceLockFence::new),
4878                Arc::clone(metadata_guard),
4879            )));
4880            Some(incarnation_id)
4881        } else {
4882            None
4883        };
4884        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
4885            && expected != current
4886        {
4887            return Err(StorageError::BucketNotFound(bucket.to_string()));
4888        }
4889        if opts.expected_bucket_incarnation_id.is_none() {
4890            opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
4891        }
4892        #[cfg(any(test, feature = "test-util"))]
4893        pause_delete_after_object_lock_snapshot(bucket).await;
4894
4895        if opts.delete_prefix && !opts.delete_prefix_object {
4896            delete_recursive_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4897            return Ok(ObjectInfo::default());
4898        }
4899
4900        let _object_lock_guard = if opts.expected_current_version_id.is_none() {
4901            self.acquire_object_write_lock_if_needed("delete_object", bucket, object, &mut opts)
4902                .await?
4903        } else {
4904            None
4905        };
4906        #[cfg(any(test, feature = "test-util"))]
4907        if _object_lock_guard.is_some() {
4908            notify_delete_namespace_acquired(bucket);
4909        }
4910        if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
4911            let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
4912            let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
4913            if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
4914                return Err(StorageError::PreconditionFailed);
4915            }
4916            let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
4917            let (mut current, _) = self
4918                .get_latest_object_info_with_idx(
4919                    bucket,
4920                    object,
4921                    &ObjectOptions {
4922                        no_lock: true,
4923                        metadata_cache_safe: false,
4924                        ..Default::default()
4925                    },
4926                )
4927                .await?;
4928            let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
4929            if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
4930                return Err(StorageError::PreconditionFailed);
4931            }
4932            current.name = decode_dir_object(&current.name);
4933            let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), &current).await;
4934            if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
4935                return Err(StorageError::PreconditionFailed);
4936            }
4937        }
4938        if opts.delete_prefix {
4939            delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4940            return Ok(ObjectInfo::default());
4941        }
4942
4943        let creates_latest_marker = latest_versioned_delete_creates_distinct_marker(&opts)
4944            || (opts.version_suspended && should_create_delete_marker_for_missing_object(&opts));
4945        let mut gopts = delete_pool_lookup_opts(&opts, true);
4946        if creates_latest_marker {
4947            // An unwritable source still owns its current version. Hiding it
4948            // during lookup would turn a rejected write into a new-pool marker.
4949            gopts.skip_rebalancing = false;
4950        }
4951
4952        if opts.data_movement {
4953            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
4954            let existing_pool_idx = existing_pool_info
4955                .as_ref()
4956                .map(|(pinfo, _)| pinfo.index)
4957                .map_err(Clone::clone);
4958            let selected_target_pool_idx =
4959                match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
4960                    Some(pool_idx) => pool_idx,
4961                    None => {
4962                        if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4963                            self.select_decommission_capacity_target_pool(owner, 0).await?
4964                        } else {
4965                            self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4966                                .await
4967                                .ok_or(Error::DiskFull)?
4968                        }
4969                    }
4970                };
4971            let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
4972                if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4973                    Some(self.select_decommission_capacity_target_pool(owner, 0).await?)
4974                } else {
4975                    self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4976                        .await
4977                }
4978            } else {
4979                None
4980            };
4981            let target_pool_idx =
4982                resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
4983            let mut delete_marker_target_opts = None;
4984
4985            if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4986                let source = self
4987                    .find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
4988                    .await?;
4989                let Some(source) = source else {
4990                    return Err(StorageError::DataMovementOverwriteErr(
4991                        bucket.to_owned(),
4992                        object.to_owned(),
4993                        opts.version_id.unwrap_or_default(),
4994                    ));
4995                };
4996                if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
4997                    return Err(StorageError::DataMovementOverwriteErr(
4998                        bucket.to_owned(),
4999                        object.to_owned(),
5000                        opts.version_id.unwrap_or_default(),
5001                    ));
5002                }
5003                let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
5004                    return Err(StorageError::DataMovementOverwriteErr(
5005                        bucket.to_owned(),
5006                        object.to_owned(),
5007                        opts.version_id.unwrap_or_default(),
5008                    ));
5009                };
5010                let target = self
5011                    .find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
5012                    .await?;
5013                if let Some(target) = target {
5014                    if is_equivalent_data_movement_delete_marker(&source, &target) {
5015                        if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) {
5016                            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
5017                                .await?;
5018                        }
5019                        let mut target = target;
5020                        target.name = decode_dir_object(object);
5021                        return Ok(target);
5022                    }
5023                    return Err(StorageError::DataMovementOverwriteErr(
5024                        bucket.to_owned(),
5025                        object.to_owned(),
5026                        opts.version_id.unwrap_or_default(),
5027                    ));
5028                }
5029                delete_marker_target_opts = Some(target_opts);
5030            }
5031
5032            if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
5033                if let Ok((source_pool_info, _)) = existing_pool_info
5034                    && opts.delete_marker
5035                    && is_data_movement_delete_marker(&source_pool_info.object_info)
5036                    && self
5037                        .has_equivalent_data_movement_delete_marker(
5038                            bucket,
5039                            object,
5040                            &source_pool_info.object_info,
5041                            &opts,
5042                            target_pool_idx,
5043                        )
5044                        .await?
5045                {
5046                    let mut obj = source_pool_info.object_info;
5047                    obj.name = decode_dir_object(object);
5048                    return Ok(obj);
5049                }
5050
5051                return Err(StorageError::DataMovementOverwriteErr(
5052                    bucket.to_owned(),
5053                    object.to_owned(),
5054                    opts.version_id.unwrap_or_default(),
5055                ));
5056            }
5057
5058            let target_opts = delete_marker_target_opts.unwrap_or(opts);
5059            let mut obj = self
5060                .run_decommission_capacity_admitted_mutation_with_capacity_lease(
5061                    target_pool_idx,
5062                    DecommissionCapacityOwner::from_options(&target_opts),
5063                    None,
5064                    |capacity_lease| async move {
5065                        let mut target_opts = target_opts;
5066                        if let Some(capacity_lease) = capacity_lease {
5067                            target_opts.add_namespace_lock_lost_signal(capacity_lease);
5068                        }
5069                        self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await
5070                    },
5071                )
5072                .await?;
5073            obj.name = decode_dir_object(obj.name.as_str());
5074            return Ok(obj);
5075        }
5076
5077        // Determine which pool contains it
5078        let existing_pool_info = if creates_latest_marker {
5079            self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
5080        } else {
5081            self.get_pool_info_existing_with_opts(bucket, object, &gopts).await
5082        };
5083        let (mut pinfo, errs) = match existing_pool_info {
5084            Ok(res) => res,
5085            Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
5086            Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
5087                let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5088                let pool = self.pools[target_pool_idx].clone();
5089                let mut obj = self
5090                    .run_external_decommission_capacity_object_mutation(
5091                        target_pool_idx,
5092                        bucket,
5093                        object,
5094                        object,
5095                        opts,
5096                        |opts| async move { pool.delete_object(bucket, object, opts).await },
5097                    )
5098                    .await?;
5099                #[cfg(test)]
5100                pause_versioned_delete_marker_after_commit(bucket, object).await;
5101                obj.name = decode_dir_object(object);
5102                return Ok(obj);
5103            }
5104            Err(err) => {
5105                // A folder key (`prefix/`) with no object metadata may still exist on
5106                // disk as an orphan empty-directory tree (issue #4189): listings show
5107                // it as a common prefix, but no regular delete path can remove it.
5108                // Purge the orphan tree so folder deletes actually take effect.
5109                if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
5110                    return Ok(ObjectInfo {
5111                        bucket: bucket.to_owned(),
5112                        name: decode_dir_object(object),
5113                        ..Default::default()
5114                    });
5115                }
5116                return Err(err);
5117            }
5118        };
5119
5120        if creates_latest_marker && self.is_suspended(pinfo.index).await {
5121            let has_active_reservation = self
5122                .pool_meta
5123                .read()
5124                .await
5125                .has_active_decommission_capacity_reservation(pinfo.index);
5126            if has_active_reservation {
5127                pinfo.index = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5128            }
5129        }
5130
5131        if pinfo.object_info.delete_marker && opts.version_id.is_none() && !creates_latest_marker {
5132            pinfo.object_info.name = decode_dir_object(object);
5133            return Ok(pinfo.object_info);
5134        }
5135
5136        if opts.data_movement && opts.src_pool_idx == pinfo.index {
5137            return Err(StorageError::DataMovementOverwriteErr(
5138                bucket.to_owned(),
5139                object.to_owned(),
5140                opts.version_id.unwrap_or_default(),
5141            ));
5142        }
5143
5144        if should_delete_from_all_pools(&opts, errs.len()) {
5145            let mut obj = self
5146                .delete_object_from_all_pools(bucket, object, &opts, &pinfo.object_info, errs)
5147                .await?;
5148            obj.name = decode_dir_object(object);
5149            return Ok(obj);
5150        }
5151
5152        for pool in self.pools.iter() {
5153            if creates_latest_marker && pool.pool_idx != pinfo.index {
5154                continue;
5155            }
5156            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
5157                if creates_latest_marker {
5158                    return Err(StorageError::SlowDown);
5159                }
5160                continue;
5161            }
5162
5163            let pool_idx = pool.pool_idx;
5164            let pool = pool.clone();
5165            match self
5166                .run_external_decommission_capacity_object_delete(
5167                    pool_idx,
5168                    bucket,
5169                    object,
5170                    object,
5171                    opts.clone(),
5172                    |opts| async move { pool.delete_object(bucket, object, opts).await },
5173                )
5174                .await
5175            {
5176                Ok(res) => {
5177                    #[cfg(test)]
5178                    pause_versioned_delete_marker_after_commit(bucket, object).await;
5179                    let mut obj = res;
5180                    obj.name = decode_dir_object(object);
5181                    return Ok(obj);
5182                }
5183                Err(err) => {
5184                    if creates_latest_marker || (!is_err_object_not_found(&err) && !is_err_version_not_found(&err)) {
5185                        return Err(err);
5186                    }
5187                }
5188            }
5189        }
5190
5191        if let Some(ver) = opts.version_id {
5192            return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
5193        }
5194
5195        Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
5196    }
5197
5198    #[instrument(skip(self, objects, opts))]
5199    pub(super) async fn handle_delete_objects(
5200        &self,
5201        bucket: &str,
5202        objects: Vec<ObjectToDelete>,
5203        opts: ObjectOptions,
5204    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5205        self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
5206    }
5207
5208    pub(super) async fn handle_delete_objects_with_journal(
5209        &self,
5210        bucket: &str,
5211        objects: Vec<ObjectToDelete>,
5212        opts: ObjectOptions,
5213        tier_journal_api: Option<Arc<ECStore>>,
5214    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5215        let (deleted, errors, _) = self
5216            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
5217            .await;
5218        (deleted, errors)
5219    }
5220
5221    pub(super) async fn handle_delete_objects_with_journal_and_accounting(
5222        &self,
5223        bucket: &str,
5224        objects: Vec<ObjectToDelete>,
5225        opts: ObjectOptions,
5226        tier_journal_api: Option<Arc<ECStore>>,
5227    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5228        Box::pin(async move {
5229            let mut opts = opts;
5230            let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
5231            let result = self
5232                .handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
5233                .await;
5234            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
5235            result
5236        })
5237        .await
5238    }
5239
5240    async fn handle_delete_objects_with_journal_and_accounting_inner(
5241        &self,
5242        bucket: &str,
5243        objects: Vec<ObjectToDelete>,
5244        opts: ObjectOptions,
5245        tier_journal_api: Option<Arc<ECStore>>,
5246    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5247        // encode object name
5248        let objects: Vec<ObjectToDelete> = objects
5249            .iter()
5250            .map(|v| {
5251                let mut v = v.clone();
5252                v.object_name = encode_dir_object(v.object_name.as_str());
5253                v
5254            })
5255            .collect();
5256
5257        // Default return value
5258        let mut del_objects = vec![DeletedObject::default(); objects.len()];
5259        let mut accounting = vec![None; objects.len()];
5260
5261        let mut del_errs = Vec::with_capacity(objects.len());
5262        for _ in 0..objects.len() {
5263            del_errs.push(None)
5264        }
5265
5266        let mut opts = opts;
5267        opts.tier_delete_journal_api = tier_journal_api;
5268        let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5269            None
5270        } else {
5271            match self.acquire_bucket_lifecycle_read_lock(bucket).await {
5272                Ok(guard) => Some(guard),
5273                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5274            }
5275        };
5276        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
5277            opts.add_bucket_lifecycle_lock_guard(guard);
5278        }
5279        if opts.delete_replication_config_snapshot.is_none() {
5280            match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
5281                Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
5282                Err(err) => {
5283                    let message = err.to_string();
5284                    let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
5285                    return (del_objects, errors, accounting);
5286                }
5287            }
5288        }
5289        if !is_meta_bucketname(bucket)
5290            && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
5291        {
5292            return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
5293        }
5294        let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
5295            None
5296        } else {
5297            Some(Arc::new(
5298                match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
5299                    Ok(guard) => guard,
5300                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5301                },
5302            ))
5303        };
5304        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
5305            opts.add_namespace_lock_guard(guard);
5306        }
5307        let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
5308            let (state, incarnation_id, config_revision) =
5309                match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
5310                    Ok(snapshot) => snapshot,
5311                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5312                };
5313            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
5314                self.id,
5315                bucket,
5316                incarnation_id,
5317                config_revision,
5318                state,
5319                opts.bucket_lifecycle_lock_fence
5320                    .clone()
5321                    .unwrap_or_else(NamespaceLockFence::new),
5322                Arc::clone(metadata_guard),
5323            )));
5324            Some(incarnation_id)
5325        } else {
5326            None
5327        };
5328        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
5329            && expected != current
5330        {
5331            return return_batch_delete_lock_error_with_accounting(
5332                objects.as_slice(),
5333                StorageError::BucketNotFound(bucket.to_string()),
5334            );
5335        }
5336        if opts.expected_bucket_incarnation_id.is_none() {
5337            opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
5338        }
5339        #[cfg(any(test, feature = "test-util"))]
5340        if current_bucket_incarnation_id.is_some() {
5341            pause_delete_after_object_lock_snapshot(bucket).await;
5342        }
5343        let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
5344            Ok(guards) => guards,
5345            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5346        };
5347        #[cfg(any(test, feature = "test-util"))]
5348        if !_object_lock_guards.is_empty() {
5349            notify_delete_namespace_acquired(bucket);
5350        }
5351
5352        let delete_config_snapshot = opts
5353            .delete_replication_config_snapshot
5354            .as_deref()
5355            .expect("batch delete replication config snapshot should be loaded");
5356        let latest_marker_objects = objects
5357            .iter()
5358            .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot))
5359            .collect::<Vec<_>>();
5360        let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map(
5361            |(object, creates_marker)| async move {
5362                if *creates_marker {
5363                    Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await)
5364                } else {
5365                    None
5366                }
5367            },
5368        ))
5369        .await;
5370        let mut marker_target_pool_indices = Vec::with_capacity(objects.len());
5371        for (idx, target_result) in marker_target_results.into_iter().enumerate() {
5372            match target_result {
5373                Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)),
5374                Some(Err(err)) => {
5375                    del_errs[idx] = Some(err);
5376                    marker_target_pool_indices.push(None);
5377                }
5378                None => marker_target_pool_indices.push(None),
5379            }
5380        }
5381
5382        let _capacity_fence = if !self.single_pool() && latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
5383            // Only marker destinations can grow. Other pools participate in
5384            // exact deletion under the same metadata read fence and must not
5385            // be treated as publication targets merely because they retire.
5386            let mut target_pool_indices = marker_target_pool_indices.iter().flatten().copied().collect::<Vec<_>>();
5387            target_pool_indices.sort_unstable();
5388            target_pool_indices.dedup();
5389            match self
5390                .acquire_external_decommission_capacity_fence(&target_pool_indices, DecommissionCapacityAdmission::BatchDelete)
5391                .await
5392            {
5393                Ok(fence) => Some(fence),
5394                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5395            }
5396        } else {
5397            None
5398        };
5399
5400        let mut futures = Vec::with_capacity(self.pools.len());
5401        for pool in self.pools.iter() {
5402            if self.is_pool_rebalancing(pool.pool_idx).await {
5403                continue;
5404            }
5405
5406            let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects
5407                .iter()
5408                .enumerate()
5409                .filter(|(idx, _)| {
5410                    batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx)
5411                })
5412                .map(|(idx, object)| (idx, object.clone()))
5413                .unzip();
5414            if pool_objects.is_empty() {
5415                continue;
5416            }
5417
5418            let pool_opts = opts.clone();
5419            futures.push(async move {
5420                #[cfg(test)]
5421                let pool_object_names = pool_objects
5422                    .iter()
5423                    .map(|object| object.object_name.clone())
5424                    .collect::<Vec<_>>();
5425                let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
5426                #[cfg(test)]
5427                let result = {
5428                    let mut result = result;
5429                    let (deleted, errors, _) = &mut result;
5430                    inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
5431                    result
5432                };
5433                (object_indices, result)
5434            });
5435        }
5436
5437        let results = join_all(futures).await;
5438
5439        for idx in 0..del_objects.len() {
5440            let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
5441                let pool_object_idx = object_indices.binary_search(&idx).ok()?;
5442                Some((&dels[pool_object_idx], &errs[pool_object_idx]))
5443            });
5444            let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results);
5445            if let Some(deleted) = deleted {
5446                del_objects[idx] = deleted;
5447            }
5448            del_errs[idx] = error;
5449
5450            if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] {
5451                del_objects[idx] = DeletedObject {
5452                    object_name: objects[idx].object_name.clone(),
5453                    version_id: objects[idx].version_id,
5454                    ..Default::default()
5455                };
5456                del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone()));
5457            }
5458        }
5459
5460        for (object_indices, (_, _, pool_accounting)) in &results {
5461            for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
5462                accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
5463            }
5464        }
5465
5466        #[cfg(test)]
5467        for (idx, object) in objects.iter().enumerate() {
5468            if del_errs[idx].is_none() && del_objects[idx].delete_marker {
5469                pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await;
5470            }
5471        }
5472
5473        del_objects.iter_mut().for_each(|v| {
5474            v.object_name = decode_dir_object(&v.object_name);
5475        });
5476
5477        (del_objects, del_errs, accounting)
5478
5479        // let mut futures = Vec::with_capacity(objects.len());
5480
5481        // for obj in objects.iter() {
5482        //     futures.push(async move {
5483        //         self.internal_get_pool_info_existing_with_opts(
5484        //             bucket,
5485        //             &obj.object_name,
5486        //             &ObjectOptions {
5487        //                 no_lock: true,
5488        //                 ..Default::default()
5489        //             },
5490        //         )
5491        //         .await
5492        //     });
5493        // }
5494
5495        // let results = join_all(futures).await;
5496
5497        // // let mut jhs = Vec::new();
5498        // // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
5499        // // let pools = Arc::new(self.pools.clone());
5500
5501        // // for obj in objects.iter() {
5502        // //     let (semaphore, pools, bucket, object_name, opt) = (
5503        // //         semaphore.clone(),
5504        // //         pools.clone(),
5505        // //         bucket.to_string(),
5506        // //         obj.object_name.to_string(),
5507        // //         ObjectOptions::default(),
5508        // //     );
5509
5510        // //     let jh = tokio::spawn(async move {
5511        // //         let _permit = semaphore.acquire().await.unwrap();
5512        // //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
5513        // //             .await
5514        // //     });
5515        // //     jhs.push(jh);
5516        // // }
5517        // // let mut results = Vec::new();
5518        // // for jh in jhs {
5519        // //     results.push(jh.await.unwrap());
5520        // // }
5521
5522        // // Record the mapping pool_idx -> object index
5523        // let mut pool_obj_idx_map = HashMap::new();
5524        // let mut orig_index_map = HashMap::new();
5525
5526        // for (i, res) in results.into_iter().enumerate() {
5527        //     match res {
5528        //         Ok((pinfo, _)) => {
5529        //             if let Some(obj) = objects.get(i) {
5530        //                 if pinfo.object_info.delete_marker && obj.version_id.is_none() {
5531        //                     del_objects[i] = DeletedObject {
5532        //                         delete_marker: pinfo.object_info.delete_marker,
5533        //                         delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
5534        //                         object_name: decode_dir_object(&pinfo.object_info.name),
5535        //                         delete_marker_mtime: pinfo.object_info.mod_time,
5536        //                         ..Default::default()
5537        //                     };
5538        //                     continue;
5539        //                 }
5540
5541        //                 if !pool_obj_idx_map.contains_key(&pinfo.index) {
5542        //                     pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
5543        //                 } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
5544        //                     val.push(obj.clone());
5545        //                 }
5546
5547        //                 if !orig_index_map.contains_key(&pinfo.index) {
5548        //                     orig_index_map.insert(pinfo.index, vec![i]);
5549        //                 } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
5550        //                     val.push(i);
5551        //                 }
5552        //             }
5553        //         }
5554        //         Err(e) => {
5555        //             if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
5556        //                 del_errs[i] = Some(e)
5557        //             }
5558
5559        //             if let Some(obj) = objects.get(i) {
5560        //                 del_objects[i] = DeletedObject {
5561        //                     object_name: decode_dir_object(&obj.object_name),
5562        //                     version_id: obj.version_id.map(|v| v.to_string()),
5563        //                     ..Default::default()
5564        //                 }
5565        //             }
5566        //         }
5567        //     }
5568        // }
5569
5570        // if !pool_obj_idx_map.is_empty() {
5571        //     for (i, sets) in self.pools.iter().enumerate() {
5572        //         // Retrieve the object index for a pool idx
5573        //         if let Some(objs) = pool_obj_idx_map.get(&i) {
5574        //             // Fetch the corresponding object (should never be None)
5575        //             // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
5576
5577        //             if objs.is_empty() {
5578        //                 continue;
5579        //             }
5580
5581        //             let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;
5582
5583        //             // Insert simultaneously (should never be None)
5584        //             let org_indexes = orig_index_map.get(&i).unwrap();
5585
5586        //             // perrs should follow the same order as obj_idxs
5587        //             for (i, err) in perrs.into_iter().enumerate() {
5588        //                 let obj_idx = org_indexes[i];
5589
5590        //                 if err.is_some() {
5591        //                     del_errs[obj_idx] = err;
5592        //                 }
5593
5594        //                 let mut dobj = pdel_objs.get(i).unwrap().clone();
5595        //                 dobj.object_name = decode_dir_object(&dobj.object_name);
5596
5597        //                 del_objects[obj_idx] = dobj;
5598        //             }
5599        //         }
5600        //     }
5601        // }
5602
5603        // Ok((del_objects, del_errs))
5604    }
5605
5606    #[instrument(skip(self))]
5607    pub(super) async fn handle_add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
5608        let object = encode_dir_object(object);
5609
5610        if self.single_pool() {
5611            let _ = self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
5612            return Ok(());
5613        }
5614
5615        let opts = ObjectOptions {
5616            version_id: Some(version_id.to_string()),
5617            ..Default::default()
5618        };
5619        let (_, idx) = self
5620            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5621            .await?;
5622
5623        let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
5624        Ok(())
5625    }
5626
5627    #[instrument(skip(self))]
5628    pub(super) async fn handle_transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5629        let object = encode_dir_object(object);
5630        if self.single_pool() {
5631            return self.pools[0].transition_object(bucket, &object, opts).await;
5632        }
5633
5634        let opts = transition_restore_pool_opts(opts);
5635        let (_, idx) = self
5636            .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
5637            .await?;
5638
5639        self.pools[idx].transition_object(bucket, &object, &opts).await
5640    }
5641
5642    #[instrument(skip(self))]
5643    pub(super) async fn handle_restore_transitioned_object(
5644        self: Arc<Self>,
5645        bucket: &str,
5646        object: &str,
5647        opts: &ObjectOptions,
5648    ) -> Result<()> {
5649        let object = encode_dir_object(object);
5650        let mut opts = transition_restore_pool_opts(opts);
5651        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
5652            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
5653        }
5654        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5655            None
5656        } else {
5657            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
5658        };
5659        if let Some(guard) = bucket_lifecycle_guard.as_ref() {
5660            opts.add_bucket_lifecycle_lock_guard(guard);
5661        }
5662        if !is_meta_bucketname(bucket) {
5663            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
5664            if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
5665                return Err(StorageError::BucketNotFound(bucket.to_string()));
5666            }
5667        }
5668        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
5669            let expected_incarnation_id = opts
5670                .expected_bucket_incarnation_id
5671                .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?;
5672            let lifecycle_fence = opts
5673                .bucket_lifecycle_lock_fence
5674                .as_ref()
5675                .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?;
5676            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
5677                Some(snapshot) => Arc::clone(snapshot),
5678                None => {
5679                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
5680                        .await?
5681                }
5682            };
5683            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
5684                return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation"));
5685            }
5686            snapshot.add_lock_fences(&mut opts);
5687            opts.object_lock_config_snapshot = Some(snapshot);
5688        }
5689        // Deliberately NOT holding the object write lock across the tier
5690        // copy-back (backlog#1304): non-SELECT restore-vs-restore is
5691        // serialized by the accept path's compare-and-set of the ongoing flag
5692        // (see acquire_restore_accept_guard), and while ongoing-request="true"
5693        // the restore header parses with no expiry, so DeleteRestoredAction
5694        // cannot fire mid-copy-back. Torn-write protection against concurrent
5695        // readers and writers stays with the inner put_object /
5696        // complete_multipart_upload commit phases, which take this object's
5697        // write lock themselves. A delete (user or lifecycle) landing between
5698        // the tier read and that commit can still be overwritten by the
5699        // commit — the same window MinIO accepts; a commit-time existence
5700        // re-check is tracked separately. Holding the lock here instead
5701        // (#4877) blocked HEAD/get_object_info for the whole copy-back and
5702        // self-deadlocked on the inner commits.
5703        let object_name = object.as_str();
5704        if self.single_pool() {
5705            return self.pools[0]
5706                .clone()
5707                .restore_transitioned_object(bucket, object_name, &opts)
5708                .await;
5709        }
5710
5711        let (_, idx) = self
5712            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock))
5713            .await?;
5714
5715        opts.decommission_capacity_admission = Some(Arc::clone(&self));
5716        self.pools[idx]
5717            .clone()
5718            .restore_transitioned_object(bucket, object_name, &opts)
5719            .await
5720    }
5721
5722    #[instrument(skip(self))]
5723    pub(super) async fn handle_put_object_metadata(
5724        &self,
5725        bucket: &str,
5726        object: &str,
5727        opts: &ObjectOptions,
5728    ) -> Result<ObjectInfo> {
5729        let object = encode_dir_object(object);
5730        let mut opts = opts.clone();
5731        opts.metadata_chg = true;
5732        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5733            None
5734        } else {
5735            let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
5736            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
5737            if opts
5738                .expected_bucket_incarnation_id
5739                .is_some_and(|expected| expected != current_incarnation_id)
5740            {
5741                return Err(StorageError::BucketNotFound(bucket.to_string()));
5742            }
5743            opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
5744            opts.add_bucket_lifecycle_lock_guard(&guard);
5745            if guard.is_lock_lost() {
5746                return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
5747            }
5748            Some(guard)
5749        };
5750
5751        if self.single_pool() {
5752            let pool = self.pools[0].clone();
5753            let capacity_object = object.clone();
5754            return self
5755                .run_external_decommission_capacity_object_mutation(
5756                    0,
5757                    bucket,
5758                    capacity_object.as_str(),
5759                    capacity_object.as_str(),
5760                    opts,
5761                    |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5762                )
5763                .await;
5764        }
5765
5766        let (_, idx) = self
5767            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5768            .await?;
5769
5770        let pool = self.pools[idx].clone();
5771        let capacity_object = object.clone();
5772        let result = self
5773            .run_external_decommission_capacity_object_mutation(
5774                idx,
5775                bucket,
5776                capacity_object.as_str(),
5777                capacity_object.as_str(),
5778                opts,
5779                |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5780            )
5781            .await;
5782        drop(bucket_lifecycle_guard);
5783        result
5784    }
5785
5786    #[instrument(skip(self))]
5787    pub(super) async fn handle_get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
5788        let object = encode_dir_object(object);
5789
5790        if self.single_pool() {
5791            return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
5792        }
5793
5794        let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
5795        Ok((*oi.user_tags).clone())
5796    }
5797
5798    #[instrument(level = "debug", skip(self))]
5799    pub(super) async fn handle_put_object_tags(
5800        &self,
5801        bucket: &str,
5802        object: &str,
5803        tags: &str,
5804        opts: &ObjectOptions,
5805    ) -> Result<ObjectInfo> {
5806        let object = encode_dir_object(object);
5807        let object_name = object.as_str();
5808
5809        if self.single_pool() {
5810            return self
5811                .run_external_decommission_capacity_object_mutation(
5812                    0,
5813                    bucket,
5814                    object_name,
5815                    object_name,
5816                    opts.clone(),
5817                    |opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await },
5818                )
5819                .await;
5820        }
5821
5822        let (_, idx) = self
5823            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5824            .await?;
5825
5826        self.run_external_decommission_capacity_object_mutation(
5827            idx,
5828            bucket,
5829            object_name,
5830            object_name,
5831            opts.clone(),
5832            |opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await },
5833        )
5834        .await
5835    }
5836
5837    #[instrument(skip(self))]
5838    pub(super) async fn handle_delete_object_version(
5839        &self,
5840        bucket: &str,
5841        object: &str,
5842        fi: &FileInfo,
5843        force_del_marker: bool,
5844    ) -> Result<()> {
5845        check_del_obj_args(bucket, object)?;
5846
5847        let object = encode_dir_object(object);
5848
5849        if self.single_pool() {
5850            return self.pools[0]
5851                .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
5852                .await;
5853        }
5854        Err(StorageError::NotImplemented)
5855    }
5856
5857    #[instrument(skip(self))]
5858    pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
5859        let object = encode_dir_object(object);
5860        let object_name = object.as_str();
5861
5862        if self.single_pool() {
5863            return self
5864                .run_external_decommission_capacity_object_mutation(
5865                    0,
5866                    bucket,
5867                    object_name,
5868                    object_name,
5869                    opts.clone(),
5870                    |opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await },
5871                )
5872                .await;
5873        }
5874
5875        let (_, idx) = self
5876            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5877            .await?;
5878
5879        self.run_external_decommission_capacity_object_mutation(
5880            idx,
5881            bucket,
5882            object_name,
5883            object_name,
5884            opts.clone(),
5885            |opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await },
5886        )
5887        .await
5888    }
5889
5890    pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5891        let get_object_reader = <Self as crate::storage_api_contracts::object::ObjectIO>::get_object_reader(
5892            self,
5893            bucket,
5894            object,
5895            None,
5896            HeaderMap::new(),
5897            opts,
5898        )
5899        .await?;
5900        // Stream to sink to avoid loading entire object into memory during verification
5901        let mut reader = get_object_reader.stream;
5902        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
5903        Ok(())
5904    }
5905}
5906
5907#[cfg(test)]
5908mod tests {
5909    use super::*;
5910    use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
5911    use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
5912    use crate::bucket::metadata_sys::ObjectLockConfigState;
5913    use crate::bucket::replication::{
5914        ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
5915        version_purge_statuses_map,
5916    };
5917    use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
5918    use crate::core::sets::make_local_two_set_sets_with_ctx;
5919    use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx};
5920    use crate::layout::{
5921        endpoints::{Endpoints, PoolEndpoints, SetupType},
5922        format::FormatV3,
5923    };
5924    use crate::object_api::{
5925        GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
5926        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
5927    };
5928    use crate::set_disk::same_distributed_lock_domain;
5929    use crate::set_disk::{SetDisks, disk_call_counters};
5930    use crate::storage_api_contracts::bucket::MakeBucketOptions;
5931    use crate::storage_api_contracts::lifecycle::TransitionedObject;
5932    use bytes::Bytes;
5933    use std::future::Future as _;
5934    use std::io::Cursor;
5935    use std::sync::Arc;
5936    use std::sync::atomic::{AtomicUsize, Ordering};
5937    use tokio::io::AsyncReadExt;
5938
5939    #[test]
5940    fn predecessor_replay_scope_distinguishes_exact_object_from_recursive_prefix() {
5941        let source = TierDeleteSourceIdentity {
5942            bucket: "bucket".to_string(),
5943            object: "directory/child".to_string(),
5944            version_id: Some("version".to_string()),
5945            versioned: true,
5946            version_suspended: false,
5947            data_dir: Some("data-dir".to_string()),
5948            etag: Some("etag".to_string()),
5949            mod_time: Some("mod-time".to_string()),
5950        };
5951        let persisted_prefix = encode_dir_object("directory/");
5952
5953        assert!(tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, false,));
5954        assert!(!tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, true,));
5955        let mut exact_source = source;
5956        exact_source.object = "directory/".to_string();
5957        assert!(tier_delete_source_matches_replay_scope(&exact_source, "bucket", &persisted_prefix, true,));
5958    }
5959
5960    struct WaitForLockLossReader {
5961        inner: Cursor<Vec<u8>>,
5962        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5963        resume: ParkingMutex<std::sync::mpsc::Receiver<()>>,
5964    }
5965
5966    impl AsyncRead for WaitForLockLossReader {
5967        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5968            let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
5969            if let Some(poll_started) = self.poll_started.take() {
5970                let _ = poll_started.send(());
5971                if self.resume.lock().recv_timeout(Duration::from_secs(10)).is_err() {
5972                    return Poll::Ready(Err(std::io::Error::new(
5973                        std::io::ErrorKind::TimedOut,
5974                        "lock-loss signal was not observed during read",
5975                    )));
5976                }
5977            }
5978            poll
5979        }
5980    }
5981
5982    struct PermanentlyPendingReader {
5983        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5984    }
5985
5986    impl AsyncRead for PermanentlyPendingReader {
5987        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5988            if let Some(poll_started) = self.poll_started.take() {
5989                let _ = poll_started.send(());
5990            }
5991            Poll::Pending
5992        }
5993    }
5994
5995    struct CommitLateBodyReader {
5996        inner: Cursor<Vec<u8>>,
5997        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5998        release: Option<tokio::sync::oneshot::Receiver<()>>,
5999    }
6000
6001    impl AsyncRead for CommitLateBodyReader {
6002        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
6003            if let Some(poll_started) = self.poll_started.take() {
6004                let _ = poll_started.send(());
6005            }
6006            if let Some(release) = self.release.as_mut() {
6007                match Pin::new(release).poll(cx) {
6008                    Poll::Pending => return Poll::Pending,
6009                    Poll::Ready(Ok(())) => self.release = None,
6010                    Poll::Ready(Err(_)) => {
6011                        return Poll::Ready(Err(std::io::Error::new(
6012                            std::io::ErrorKind::BrokenPipe,
6013                            "commit-late body release sender was dropped",
6014                        )));
6015                    }
6016                }
6017            }
6018            Pin::new(&mut self.inner).poll_read(cx, buf)
6019        }
6020    }
6021
6022    struct CountingMissHook {
6023        calls: AtomicUsize,
6024    }
6025
6026    async fn refresh_failure_test_guard(
6027        owner: &'static str,
6028    ) -> (
6029        ObjectLockDiagGuard,
6030        Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6031        Arc<RefreshLossLockClient>,
6032    ) {
6033        let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
6034            rustfs_lock::FastObjectLockManager::new(),
6035        )));
6036        let client = Arc::new(RefreshLossLockClient::with_manager(manager));
6037        let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
6038            owner.to_string(),
6039            vec![Arc::clone(&client) as Arc<dyn rustfs_lock::LockClient>],
6040            1,
6041        );
6042        let request =
6043            rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Shared, owner)
6044                .with_ttl(Duration::from_secs(2))
6045                .with_refresh_interval(Duration::from_millis(20));
6046        let guard = namespace_lock
6047            .acquire_guard(&request)
6048            .await
6049            .expect("distributed read lock acquisition should succeed")
6050            .expect("distributed read lock should reach quorum");
6051        let guard = ObjectLockDiagGuard::new(
6052            guard,
6053            false,
6054            "SelectObjectContent",
6055            Some("bucket".to_string()),
6056            Some("object".to_string()),
6057            Some(owner.to_string()),
6058            ObjectLockDiagMode::Read,
6059        );
6060        let signal = guard
6061            .lock_lost_signal()
6062            .expect("a distributed guard should expose a lock-loss signal");
6063        (guard, signal, client)
6064    }
6065
6066    async fn refresh_failure_test_lease(
6067        owner: &'static str,
6068    ) -> (
6069        Arc<SelectObjectSnapshotLease>,
6070        Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6071        Arc<RefreshLossLockClient>,
6072    ) {
6073        let (guard, signal, client) = refresh_failure_test_guard(owner).await;
6074        (Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client)
6075    }
6076
6077    #[async_trait::async_trait]
6078    impl GetObjectBodyCacheHook for CountingMissHook {
6079        async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
6080            self.calls.fetch_add(1, Ordering::Relaxed);
6081            None
6082        }
6083    }
6084
6085    struct BodyCacheHookGuard;
6086
6087    #[test]
6088    fn select_snapshot_deduplicates_same_distributed_lock_domain() {
6089        let first: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6090        let second: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6091        let other: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6092
6093        assert!(same_distributed_lock_domain(
6094            &[Arc::clone(&first), Arc::clone(&second)],
6095            &[Arc::clone(&second), Arc::clone(&first)]
6096        ));
6097        assert!(same_distributed_lock_domain(
6098            &[Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)],
6099            &[Arc::clone(&second), Arc::clone(&first)]
6100        ));
6101        assert!(!same_distributed_lock_domain(&[first, second], &[other]));
6102    }
6103
6104    #[tokio::test]
6105    async fn publication_commit_locks_dedupe_dist_sets_with_same_clients() {
6106        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6107        let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6108        let mut second_set = (*original_sets.disk_set[1]).clone();
6109        second_set.lockers = original_sets.disk_set[0].lockers.clone();
6110        let mut sets = (*original_sets).clone();
6111        sets.disk_set[1] = Arc::new(second_set);
6112        let sets = Arc::new(sets);
6113        ctx.update_erasure_type(SetupType::DistErasure).await;
6114
6115        assert!(
6116            sets.disk_set[0]
6117                .lockers
6118                .iter()
6119                .zip(&sets.disk_set[1].lockers)
6120                .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)),
6121            "the regression requires identical distributed lock clients"
6122        );
6123        assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index);
6124
6125        let pool_config = sets.endpoints.clone();
6126        let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6127        let object = (0..1_000)
6128            .map(|index| format!("decommission-dist-domain-{index}.bin"))
6129            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6130            .expect("a key should hash to the second set namespace");
6131        let publication_guards = store
6132            .acquire_data_movement_publication_write_locks("bucket", &encode_dir_object(&object), 0, 0, true)
6133            .await
6134            .expect("the publication commit lock set should be acquired");
6135        let target_lock = sets.disk_set[1]
6136            .new_ns_lock("bucket", &object)
6137            .await
6138            .expect("the hashed-set namespace lock should be created");
6139        let target_err = target_lock
6140            .get_write_lock(Duration::from_millis(50))
6141            .await
6142            .expect_err("the fixed write fence must conflict through the shared clients");
6143        assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. }));
6144
6145        drop(publication_guards);
6146
6147        let cleanup_object = (0..1_000)
6148            .map(|index| format!("decommission-dist-cleanup-{index}.bin"))
6149            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6150            .expect("a cleanup key should hash to the second set namespace");
6151        let source_fence = store
6152            .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref())
6153            .await
6154            .expect("the fixed distributed cleanup fence should be acquired");
6155        assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence");
6156        let source_lock = sets.disk_set[1]
6157            .new_ns_lock("bucket", &cleanup_object)
6158            .await
6159            .expect("the source-set namespace lock should be created");
6160        let source_err = source_lock
6161            .get_read_lock(Duration::from_millis(50))
6162            .await
6163            .expect_err("the fixed write fence must conflict through the shared clients");
6164        assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. }));
6165    }
6166
6167    #[tokio::test]
6168    async fn physical_absence_proof_blocks_decommission_remote_tuple_publication() {
6169        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6170        let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6171        ctx.update_erasure_type(SetupType::DistErasure).await;
6172        let pool_config = sets.endpoints.clone();
6173        let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6174        let bucket = "remote-tuple-publication-fence";
6175        let object = "transitioned.bin";
6176        let encoded = encode_dir_object(object);
6177        let mut proof_opts = ObjectOptions::default();
6178        let proof = store
6179            .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6180            .await
6181            .expect("the all-physical absence proof should acquire every read domain");
6182
6183        // Keep polling the same distributed-lock request after the bounded
6184        // observation below.  Dropping a timed-out request can race with the
6185        // lock service granting it and would turn the test itself into an
6186        // abandoned-owner scenario.
6187        let mut publication = Box::pin(store.acquire_data_movement_publication_write_locks(bucket, &encoded, 0, 0, true));
6188        let blocked = tokio::time::timeout(Duration::from_millis(50), &mut publication).await;
6189        assert!(
6190            blocked.is_err(),
6191            "a decommission publisher must remain blocked while the remote-delete absence proof is held"
6192        );
6193
6194        drop(proof);
6195        publication
6196            .await
6197            .expect("the publisher should acquire its write fence after the proof releases");
6198    }
6199
6200    #[cfg(feature = "test-util")]
6201    #[tokio::test]
6202    async fn restored_transitioned_rebalance_revalidates_source_after_recovery_lock() {
6203        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6204        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6205        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6206        let store = Arc::new(
6207            new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], Arc::clone(&ctx)).await,
6208        );
6209        let bucket = "restored-transitioned-rebalance-publication";
6210        let object = "restored.bin";
6211        for set in [&first_set, &second_set] {
6212            set.make_bucket(bucket, &MakeBucketOptions::default())
6213                .await
6214                .expect("publication-race bucket should be created");
6215        }
6216
6217        let tier_name = "RESTORED-REBALANCE";
6218        crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), tier_name).await;
6219        let lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), tier_name)
6220            .await
6221            .expect("mock tier generation should be available");
6222        let backend_identity = lease.backend_identity();
6223        drop(lease);
6224
6225        let mut metadata = HashMap::new();
6226        for (suffix, value) in [
6227            (rustfs_utils::http::SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.to_string()),
6228            (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/restored.bin".to_string()),
6229            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
6230            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-version".to_string()),
6231            (
6232                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6233                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
6234            ),
6235            (
6236                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6237                rustfs_utils::crypto::hex(backend_identity),
6238            ),
6239        ] {
6240            rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, value);
6241        }
6242        metadata.insert(
6243            "x-amz-restore".to_string(),
6244            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
6245        );
6246        let mut body = PutObjReader::from_vec(b"restored local body".to_vec());
6247        first_set
6248            .put_object(
6249                bucket,
6250                object,
6251                &mut body,
6252                &ObjectOptions {
6253                    user_defined: metadata,
6254                    no_lock: true,
6255                    ..Default::default()
6256                },
6257            )
6258            .await
6259            .expect("restored-transitioned source should be written");
6260        let source = first_set
6261            .get_object_info(
6262                bucket,
6263                object,
6264                &ObjectOptions {
6265                    include_part_checksums: true,
6266                    no_lock: true,
6267                    ..Default::default()
6268                },
6269            )
6270            .await
6271            .expect("restored-transitioned source metadata should be readable");
6272        assert_eq!(source.transitioned_object.status, TRANSITION_COMPLETE);
6273        assert!(source.data_dir.is_some(), "fixture must retain restored local data");
6274
6275        let publisher_first = store
6276            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6277            .await
6278            .expect("rebalance should capture an exact publication capability");
6279        let encoded = encode_dir_object(object);
6280        let publisher_guard = publisher_first
6281            .into_commit_guard(1, bucket, &encoded)
6282            .await
6283            .expect("unchanged source should validate at the publication boundary");
6284        let mut publisher_first_proof_opts = ObjectOptions::default();
6285        let mut publisher_first_proof = Box::pin(store.acquire_all_physical_object_read_locks(
6286            "tier_delete_journal_recovery",
6287            bucket,
6288            &encoded,
6289            &mut publisher_first_proof_opts,
6290        ));
6291        assert!(
6292            tokio::time::timeout(Duration::from_millis(50), &mut publisher_first_proof)
6293                .await
6294                .is_err(),
6295            "recovery must wait while a validated publisher owns the commit scope"
6296        );
6297        drop(publisher_guard);
6298        drop(
6299            publisher_first_proof
6300                .await
6301                .expect("recovery proof should acquire after publisher commit scope releases"),
6302        );
6303
6304        let publication = store
6305            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6306            .await
6307            .expect("recovery-first race should capture the old source identity");
6308
6309        // The pre-copy capability itself must not hold a namespace write lock.
6310        let pre_commit_reader = first_set
6311            .new_ns_lock(bucket, &encoded)
6312            .await
6313            .expect("source read lock should be created")
6314            .get_read_lock(Duration::from_millis(50))
6315            .await
6316            .expect("body staging must not be blocked by an early publication write lock");
6317        drop(pre_commit_reader);
6318
6319        first_set
6320            .delete_object(
6321                bucket,
6322                &encoded,
6323                ObjectOptions {
6324                    delete_prefix: true,
6325                    delete_prefix_object: true,
6326                    data_movement: true,
6327                    no_lock: true,
6328                    ..Default::default()
6329                },
6330            )
6331            .await
6332            .expect("source cleanup should remove the old physical owner");
6333
6334        let mut proof_opts = ObjectOptions::default();
6335        let proof = store
6336            .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6337            .await
6338            .expect("recovery should hold its all-physical absence proof");
6339        let mut commit = Box::pin(publication.into_commit_guard(1, bucket, &encoded));
6340        assert!(
6341            tokio::time::timeout(Duration::from_millis(50), &mut commit).await.is_err(),
6342            "publisher must wait while recovery owns the physical absence proof"
6343        );
6344        drop(proof);
6345        let err = match commit.await {
6346            Ok(_) => panic!("publisher must reject its stale source snapshot after recovery releases"),
6347            Err(err) => err,
6348        };
6349        assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6350        assert!(
6351            second_set
6352                .get_object_info(bucket, object, &ObjectOptions::default())
6353                .await
6354                .is_err(),
6355            "stale publication must not create a target owner"
6356        );
6357    }
6358
6359    #[tokio::test]
6360    async fn data_movement_put_defers_publication_locks_until_body_complete() {
6361        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6362        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6363        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6364        let store =
6365            Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6366        let bucket = "commit-late-data-movement-put";
6367        let object = "slow-body.bin";
6368        for set in [&first_set, &second_set] {
6369            set.make_bucket(bucket, &MakeBucketOptions::default())
6370                .await
6371                .expect("commit-late PUT bucket should be created");
6372        }
6373
6374        let payload = b"data movement body".to_vec();
6375        let mut source_body = PutObjReader::from_vec(payload.clone());
6376        first_set
6377            .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6378            .await
6379            .expect("data-movement source should be written");
6380        let source = first_set
6381            .get_object_info(bucket, object, &ObjectOptions::default())
6382            .await
6383            .expect("data-movement source should be readable");
6384        let publication = store
6385            .acquire_remote_tuple_publication_fence(bucket, 0, &source, true)
6386            .await
6387            .expect("decommission should capture its publication capability");
6388
6389        let payload_len = i64::try_from(payload.len()).expect("test payload length should fit in i64");
6390        let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6391        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6392        let slow_body = CommitLateBodyReader {
6393            inner: Cursor::new(payload),
6394            poll_started: Some(poll_started_tx),
6395            release: Some(release_rx),
6396        };
6397        let target_set = Arc::clone(&store.pools[1]);
6398        let target_opts = ObjectOptions {
6399            data_movement: true,
6400            src_pool_idx: 0,
6401            http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
6402            mod_time: source.mod_time,
6403            preserve_etag: source.etag.clone(),
6404            user_defined: source.user_defined.as_ref().clone(),
6405            ..Default::default()
6406        };
6407        let worker = tokio::spawn(async move {
6408            let hash_reader = rustfs_rio::HashReader::from_stream(slow_body, payload_len, payload_len, None, None, false)
6409                .expect("slow body should produce a valid hash reader");
6410            let mut reader = PutObjReader::new(hash_reader);
6411            target_set
6412                .put_object_with_old_current_size_for_data_movement(bucket, object, &mut reader, &target_opts, publication)
6413                .await
6414        });
6415
6416        tokio::time::timeout(Duration::from_secs(5), poll_started_rx)
6417            .await
6418            .expect("target PUT should start polling its body")
6419            .expect("target PUT should retain the poll observer");
6420        let source_head = tokio::time::timeout(
6421            Duration::from_millis(250),
6422            first_set.get_object_info(bucket, object, &ObjectOptions::default()),
6423        )
6424        .await
6425        .expect("same-key HEAD must not wait behind body staging")
6426        .expect("the source must remain readable during body staging");
6427        assert_eq!(source_head.etag, source.etag);
6428        let target_head = tokio::time::timeout(
6429            Duration::from_millis(250),
6430            second_set.get_object_info(bucket, object, &ObjectOptions::default()),
6431        )
6432        .await
6433        .expect("target HEAD must not wait behind body staging")
6434        .expect_err("the target must remain unpublished before body EOF");
6435        assert!(is_err_object_not_found(&target_head) || is_err_version_not_found(&target_head));
6436
6437        release_tx
6438            .send(())
6439            .expect("target PUT should still be waiting on the body gate");
6440        worker
6441            .await
6442            .expect("target PUT task should join")
6443            .expect("target PUT should commit after body EOF");
6444        second_set
6445            .get_object_info(bucket, object, &ObjectOptions::default())
6446            .await
6447            .expect("target metadata should be visible after the commit boundary");
6448    }
6449
6450    #[test]
6451    fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() {
6452        let protected_suffixes = [
6453            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6454            rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
6455            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
6456            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6457            rustfs_utils::http::SUFFIX_TRANSITION_TIER,
6458            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6459            rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
6460            rustfs_utils::http::SUFFIX_FREE_VERSION,
6461            rustfs_utils::http::SUFFIX_TIER_FV_ID,
6462            rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
6463        ];
6464        let mut source = HashMap::from([("content-type".to_string(), "application/octet-stream".to_string())]);
6465        for suffix in protected_suffixes {
6466            rustfs_utils::http::metadata_compat::insert_str(&mut source, suffix, format!("source-{suffix}"));
6467        }
6468        // Compatibility readers are case-insensitive, so the scrubber must
6469        // remove non-canonical casing as well as both canonical prefixes.
6470        source.insert(
6471            format!(
6472                "{}{}",
6473                rustfs_utils::http::MINIO_INTERNAL_PREFIX.to_ascii_uppercase(),
6474                rustfs_utils::http::SUFFIX_TRANSITION_STATUS.to_ascii_uppercase()
6475            ),
6476            "complete".to_string(),
6477        );
6478
6479        let copied = materialized_copy_user_defined(&source);
6480
6481        assert_eq!(copied.get("content-type").map(String::as_str), Some("application/octet-stream"));
6482        for suffix in protected_suffixes {
6483            assert!(
6484                !rustfs_utils::http::metadata_compat::contains_key_str(&copied, suffix),
6485                "materialized copy retained protected internal suffix {suffix}"
6486            );
6487        }
6488    }
6489
6490    #[test]
6491    fn remote_tuple_publication_target_is_exact_legacy_compatible_and_ordinary_zero_cost() {
6492        let mut ordinary_metadata = HashMap::new();
6493        rustfs_utils::http::metadata_compat::insert_str(
6494            &mut ordinary_metadata,
6495            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6496            "malformed-but-irrelevant".to_string(),
6497        );
6498        let ordinary = ObjectInfo {
6499            user_defined: Arc::new(ordinary_metadata),
6500            ..Default::default()
6501        };
6502        assert!(
6503            remote_tuple_publication_target(&ordinary)
6504                .expect("ordinary objects must not parse or lock tier metadata")
6505                .is_none()
6506        );
6507
6508        let identity = [0xabu8; 32];
6509        let mut exact_metadata = HashMap::new();
6510        rustfs_utils::http::metadata_compat::insert_str(
6511            &mut exact_metadata,
6512            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6513            rustfs_utils::crypto::hex(identity),
6514        );
6515        let transitioned = ObjectInfo {
6516            transitioned_object: TransitionedObject {
6517                status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
6518                name: "remote/object".to_string(),
6519                tier: "WARM".to_string(),
6520                ..Default::default()
6521            },
6522            user_defined: Arc::new(exact_metadata),
6523            ..Default::default()
6524        };
6525        let (tier, recorded_identity) = remote_tuple_publication_target(&transitioned)
6526            .expect("exact transitioned metadata should parse")
6527            .expect("transitioned object should require a publication fence");
6528        assert_eq!(tier, "WARM");
6529        assert_eq!(recorded_identity, Some(identity));
6530
6531        let legacy = ObjectInfo {
6532            user_defined: Arc::new(HashMap::new()),
6533            ..transitioned.clone()
6534        };
6535        assert_eq!(
6536            remote_tuple_publication_target(&legacy)
6537                .expect("legacy transitioned metadata should remain compatible")
6538                .map(|(_, identity)| identity),
6539            Some(None)
6540        );
6541
6542        let mut malformed = transitioned;
6543        malformed.transitioned_object.tier.clear();
6544        assert!(remote_tuple_publication_target(&malformed).is_err());
6545    }
6546
6547    #[test]
6548    fn publication_source_match_rejects_new_physical_generation_with_same_logical_identity() {
6549        let expected = ObjectInfo {
6550            data_dir: Some(Uuid::new_v4()),
6551            ..Default::default()
6552        };
6553        assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6554
6555        let current = ObjectInfo {
6556            data_dir: Some(Uuid::new_v4()),
6557            ..expected.clone()
6558        };
6559        assert!(
6560            crate::data_movement::is_equivalent_data_movement_object_identity(&expected, &current, true, true),
6561            "the logical target-equivalence check deliberately ignores target data-dir allocation"
6562        );
6563        assert!(
6564            !remote_tuple_publication_object_source_matches(&expected, &current),
6565            "publication revalidation must additionally bind the physical source generation"
6566        );
6567    }
6568
6569    #[test]
6570    fn exact_delete_capacity_target_requires_matching_namespace_and_object_identity() {
6571        let source = ObjectInfo {
6572            bucket: "bucket".to_string(),
6573            name: "directory/".to_string(),
6574            version_id: Some(Uuid::from_u128(1)),
6575            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
6576            size: 10,
6577            etag: Some("source-etag".to_string()),
6578            ..Default::default()
6579        };
6580        let target = ObjectInfo {
6581            name: rustfs_utils::path::encode_dir_object(&source.name),
6582            ..source.clone()
6583        };
6584        assert!(ECStore::is_equivalent_decommission_capacity_target(&source, &target));
6585
6586        let mismatched_identity = ObjectInfo {
6587            etag: Some("different-etag".to_string()),
6588            ..target.clone()
6589        };
6590        assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &mismatched_identity));
6591
6592        let wrong_bucket = ObjectInfo {
6593            bucket: "other-bucket".to_string(),
6594            ..target
6595        };
6596        assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &wrong_bucket));
6597    }
6598
6599    fn publication_part(number: usize) -> ObjectPartInfo {
6600        ObjectPartInfo {
6601            number,
6602            size: 100 + number,
6603            actual_size: i64::try_from(200 + number).expect("test part size should fit in i64"),
6604            etag: format!("part-etag-{number}"),
6605            ..Default::default()
6606        }
6607    }
6608
6609    fn publication_source(parts: Vec<ObjectPartInfo>) -> ObjectInfo {
6610        ObjectInfo {
6611            data_dir: Some(Uuid::new_v4()),
6612            version_id: Some(Uuid::new_v4()),
6613            size: 4096,
6614            actual_size: 4096,
6615            etag: Some("object-etag".to_string()),
6616            checksum: Some(Bytes::from_static(b"object-checksum")),
6617            mod_time: Some(time::OffsetDateTime::UNIX_EPOCH),
6618            parts: Arc::new(parts),
6619            ..Default::default()
6620        }
6621    }
6622
6623    #[test]
6624    fn publication_parts_match_is_order_independent_and_bijective() {
6625        let ordered = vec![publication_part(1), publication_part(2), publication_part(3)];
6626        let mut reversed = ordered.clone();
6627        reversed.reverse();
6628
6629        assert!(remote_tuple_publication_parts_match(&[], &[]));
6630        assert!(remote_tuple_publication_parts_match(&ordered, &ordered));
6631        assert!(remote_tuple_publication_parts_match(&ordered, &reversed));
6632        assert!(!remote_tuple_publication_parts_match(&ordered[..2], &ordered));
6633
6634        let expected_duplicate = vec![publication_part(1), publication_part(1)];
6635        let current_unique = vec![publication_part(1), publication_part(2)];
6636        assert!(
6637            !remote_tuple_publication_parts_match(&expected_duplicate, &current_unique),
6638            "two expected entries must not reuse the same current part"
6639        );
6640        assert!(!remote_tuple_publication_parts_match(&current_unique, &expected_duplicate));
6641        assert!(!remote_tuple_publication_parts_match(&expected_duplicate, &expected_duplicate));
6642    }
6643
6644    #[test]
6645    fn publication_parts_match_preserves_exact_part_identity_contract() {
6646        let expected = vec![publication_part(1)];
6647
6648        let mut different_number = expected.clone();
6649        different_number[0].number = 2;
6650        assert!(!remote_tuple_publication_parts_match(&expected, &different_number));
6651
6652        let mut different_size = expected.clone();
6653        different_size[0].size += 1;
6654        assert!(!remote_tuple_publication_parts_match(&expected, &different_size));
6655
6656        let mut different_actual_size = expected.clone();
6657        different_actual_size[0].actual_size += 1;
6658        assert!(!remote_tuple_publication_parts_match(&expected, &different_actual_size));
6659
6660        let mut different_etag = expected.clone();
6661        different_etag[0].etag.push_str("-changed");
6662        assert!(!remote_tuple_publication_parts_match(&expected, &different_etag));
6663
6664        let mut ignored_fields = expected.clone();
6665        ignored_fields[0].index = Some(Bytes::from_static(b"different-index"));
6666        ignored_fields[0].checksums = Some(HashMap::from([("CRC32C".to_string(), "different".to_string())]));
6667        ignored_fields[0].mod_time = Some(time::OffsetDateTime::UNIX_EPOCH);
6668        assert!(
6669            remote_tuple_publication_parts_match(&expected, &ignored_fields),
6670            "the publication fence must retain its existing checksum/index/mod-time compatibility contract"
6671        );
6672
6673        let zero_actual_size = vec![ObjectPartInfo {
6674            actual_size: 0,
6675            ..publication_part(1)
6676        }];
6677        assert!(
6678            !remote_tuple_publication_parts_match(&zero_actual_size, &expected),
6679            "the publication fence compares raw part actual sizes without the broader comparator's fallback"
6680        );
6681    }
6682
6683    #[test]
6684    fn publication_source_match_rejects_object_etag_and_identity_mutations() {
6685        let expected = publication_source(vec![publication_part(1)]);
6686        assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6687
6688        let mut current = expected.clone();
6689        current.etag = Some("changed-object-etag".to_string());
6690        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6691
6692        let mut current = expected.clone();
6693        current.checksum = Some(Bytes::from_static(b"changed-checksum"));
6694        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6695
6696        let mut current = expected.clone();
6697        current.actual_size += 1;
6698        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6699
6700        let mut current = expected.clone();
6701        current.data_dir = Some(Uuid::new_v4());
6702        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6703
6704        let mut current = expected.clone();
6705        current.version_id = Some(Uuid::new_v4());
6706        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6707
6708        let mut current = expected.clone();
6709        current.mod_time = current.mod_time.map(|mod_time| mod_time + time::Duration::SECOND);
6710        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6711    }
6712
6713    #[test]
6714    fn publication_source_match_preserves_effective_actual_size_compatibility() {
6715        let expected = publication_source(vec![publication_part(1)]);
6716        let current = ObjectInfo {
6717            actual_size: 0,
6718            ..expected.clone()
6719        };
6720
6721        assert_eq!(
6722            expected.get_actual_size().expect("expected size should be valid"),
6723            current.get_actual_size().expect("current size should be valid")
6724        );
6725        assert!(remote_tuple_publication_object_source_matches(&expected, &current));
6726    }
6727
6728    #[test]
6729    fn publication_source_match_rejects_duplicate_parts_through_the_full_fence() {
6730        let expected = publication_source(vec![publication_part(1), publication_part(1)]);
6731        let current = ObjectInfo {
6732            parts: Arc::new(vec![publication_part(1), publication_part(2)]),
6733            ..expected.clone()
6734        };
6735
6736        assert!(
6737            !remote_tuple_publication_object_source_matches(&expected, &current),
6738            "the full source fence must reject the original non-bijective false-positive"
6739        );
6740    }
6741
6742    #[test]
6743    fn publication_source_match_handles_10_000_reversed_parts_without_payload_cloning() {
6744        let parts = Arc::new((1..=10_000).map(publication_part).collect::<Vec<_>>());
6745        let mut reversed = parts.as_ref().clone();
6746        reversed.reverse();
6747        let expected = publication_source(Vec::new());
6748        let expected = ObjectInfo {
6749            parts: Arc::clone(&parts),
6750            ..expected
6751        };
6752        let current = ObjectInfo {
6753            parts: Arc::new(reversed),
6754            ..expected.clone()
6755        };
6756        let expected_parts = Arc::clone(&expected.parts);
6757
6758        assert!(remote_tuple_publication_object_source_matches(&expected, &current));
6759        assert!(Arc::ptr_eq(&expected.parts, &expected_parts));
6760    }
6761
6762    #[tokio::test]
6763    async fn publication_commit_guard_rejects_an_etag_only_source_change() {
6764        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6765        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6766        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6767        let store =
6768            Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6769        let bucket = "publication-etag-source-change";
6770        let object = "source.bin";
6771        for set in [&first_set, &second_set] {
6772            set.make_bucket(bucket, &MakeBucketOptions::default())
6773                .await
6774                .expect("publication ETag test bucket should be created");
6775        }
6776
6777        let mut source_body = PutObjReader::from_vec(b"source body".to_vec());
6778        first_set
6779            .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6780            .await
6781            .expect("publication source should be written");
6782        let source = first_set
6783            .get_object_info(
6784                bucket,
6785                object,
6786                &ObjectOptions {
6787                    include_part_checksums: true,
6788                    ..Default::default()
6789                },
6790            )
6791            .await
6792            .expect("publication source should be readable");
6793        let publication = store
6794            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6795            .await
6796            .expect("the source snapshot should produce a publication capability");
6797
6798        let changed = first_set
6799            .put_object_metadata(
6800                bucket,
6801                object,
6802                &ObjectOptions {
6803                    eval_metadata: Some(HashMap::from([("etag".to_string(), "changed-etag".to_string())])),
6804                    ..Default::default()
6805                },
6806            )
6807            .await
6808            .expect("the source ETag should be updated in place");
6809        assert_ne!(source.etag, changed.etag);
6810        let unchanged_except_etag = ObjectInfo {
6811            etag: source.etag.clone(),
6812            ..changed.clone()
6813        };
6814        assert!(
6815            remote_tuple_publication_object_source_matches(&source, &unchanged_except_etag),
6816            "the persisted source update fixture must differ only by object ETag"
6817        );
6818
6819        let encoded = encode_dir_object(object);
6820        let err = match publication.into_commit_guard(1, bucket, &encoded).await {
6821            Ok(_) => panic!("the publication guard must reject an ETag-only source change"),
6822            Err(err) => err,
6823        };
6824        assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6825        assert!(
6826            second_set
6827                .get_object_info(bucket, object, &ObjectOptions::default())
6828                .await
6829                .is_err(),
6830            "a rejected publication must not create a target object"
6831        );
6832    }
6833
6834    #[tokio::test]
6835    async fn generic_data_movement_put_rejects_transition_ownership_without_capability() {
6836        let (_dirs, set) = make_local_set_disks(4, 2).await;
6837        let bucket = "unfenced-transition-publication";
6838        set.make_bucket(bucket, &MakeBucketOptions::default())
6839            .await
6840            .expect("unfenced publication bucket should be created");
6841        let mut metadata = HashMap::new();
6842        rustfs_utils::http::metadata_compat::insert_str(
6843            &mut metadata,
6844            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6845            TRANSITION_COMPLETE.to_string(),
6846        );
6847        let mut body = PutObjReader::from_vec(b"must not publish".to_vec());
6848        let err = set
6849            .put_object(
6850                bucket,
6851                "unfenced.bin",
6852                &mut body,
6853                &ObjectOptions {
6854                    data_movement: true,
6855                    user_defined: metadata,
6856                    ..Default::default()
6857                },
6858            )
6859            .await
6860            .expect_err("transition ownership must require the typed publication entry point");
6861        assert!(err.to_string().contains("publication capability"));
6862    }
6863
6864    #[test]
6865    fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() {
6866        let nil = Uuid::nil();
6867        for actual in [None, Some(nil)] {
6868            assert!(select_snapshot_version_matches(actual, "null"));
6869            assert!(select_snapshot_version_matches(actual, "NULL"));
6870            assert!(select_snapshot_version_matches(actual, &nil.to_string()));
6871        }
6872
6873        let version = Uuid::new_v4();
6874        assert!(select_snapshot_version_matches(Some(version), &version.to_string().to_uppercase()));
6875        assert!(!select_snapshot_version_matches(Some(version), "null"));
6876        assert!(!select_snapshot_version_matches(Some(version), "not-a-version"));
6877    }
6878
6879    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6880    async fn select_snapshot_reader_rolls_back_bytes_when_lock_is_lost_during_poll() {
6881        let (lease, signal, client) = refresh_failure_test_lease("select-snapshot-lock-loss").await;
6882
6883        let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6884        let (resume_tx, resume_rx) = std::sync::mpsc::channel();
6885        let release_client = Arc::clone(&client);
6886        let release_lease = Arc::clone(&lease);
6887        let release_signal = Arc::clone(&signal);
6888        let release_task = tokio::spawn(async move {
6889            poll_started_rx.await.expect("reader poll should start");
6890            release_client.reject_refreshes();
6891            release_client
6892                .wait_for_rejected_refresh(Duration::from_secs(5))
6893                .await
6894                .expect("refresh rejection should be observed");
6895            tokio::time::timeout(Duration::from_secs(5), release_signal.notified())
6896                .await
6897                .expect("heartbeat should observe the rejected refresh");
6898            tokio::time::timeout(Duration::from_secs(5), async {
6899                while !release_lease.is_lost() {
6900                    tokio::task::yield_now().await;
6901                }
6902            })
6903            .await
6904            .expect("snapshot monitor should publish lock loss");
6905            resume_tx.send(()).expect("reader poll should still be waiting");
6906        });
6907
6908        let mut reader = SelectObjectSnapshotReader {
6909            inner: Box::new(WaitForLockLossReader {
6910                inner: Cursor::new(b"must-not-escape".to_vec()),
6911                poll_started: Some(poll_started_tx),
6912                resume: ParkingMutex::new(resume_rx),
6913            }),
6914            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6915            lease,
6916        };
6917        let mut output = Vec::new();
6918        let error = reader
6919            .read_to_end(&mut output)
6920            .await
6921            .expect_err("a read that loses its lease must fail");
6922        release_task.await.expect("backend release task should not panic");
6923
6924        assert!(signal.is_lost(), "heartbeat should report the rejected refresh");
6925        assert!(output.is_empty(), "bytes read during the failed poll must be rolled back");
6926        assert_eq!(error.kind(), std::io::ErrorKind::Other);
6927    }
6928
6929    #[tokio::test(flavor = "current_thread")]
6930    async fn select_snapshot_reader_checks_guards_before_monitor_runs() {
6931        let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await;
6932        client.reject_refreshes();
6933        client
6934            .wait_for_rejected_refresh(Duration::from_secs(5))
6935            .await
6936            .expect("refresh rejection should be observed");
6937        tokio::time::timeout(Duration::from_secs(5), signal.notified())
6938            .await
6939            .expect("heartbeat should observe the rejected refresh");
6940
6941        let lease = Arc::new(SelectObjectSnapshotLease::new(vec![guard]));
6942        assert!(!lease.is_lost(), "current-thread monitor must not run before the synchronous read");
6943        let mut reader = SelectObjectSnapshotReader {
6944            inner: Box::new(Cursor::new(b"old-generation".to_vec())),
6945            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6946            lease,
6947        };
6948        let mut output = Vec::new();
6949
6950        let error = reader
6951            .read_to_end(&mut output)
6952            .await
6953            .expect_err("EOF fence must reject a lease lost before its monitor is scheduled");
6954
6955        assert!(output.is_empty(), "bytes from a known-lost snapshot must not escape");
6956        assert_eq!(error.kind(), std::io::ErrorKind::Other);
6957    }
6958
6959    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6960    async fn select_snapshot_lock_loss_is_broadcast_to_remaining_pending_readers() {
6961        let (first_guard, first_signal, first_client) = refresh_failure_test_guard("select-snapshot-pending-first-lock").await;
6962        let (second_guard, second_signal, second_client) =
6963            refresh_failure_test_guard("select-snapshot-pending-second-lock").await;
6964        let lease = Arc::new(SelectObjectSnapshotLease::new(vec![first_guard, second_guard]));
6965        let dropped_reader = SelectObjectSnapshotReader {
6966            inner: Box::new(PermanentlyPendingReader { poll_started: None }),
6967            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6968            lease: Arc::clone(&lease),
6969        };
6970        drop(dropped_reader);
6971
6972        let (first_started_tx, first_started_rx) = tokio::sync::oneshot::channel();
6973        let (second_started_tx, second_started_rx) = tokio::sync::oneshot::channel();
6974        let mut first_reader = SelectObjectSnapshotReader {
6975            inner: Box::new(PermanentlyPendingReader {
6976                poll_started: Some(first_started_tx),
6977            }),
6978            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6979            lease: Arc::clone(&lease),
6980        };
6981        let mut second_reader = SelectObjectSnapshotReader {
6982            inner: Box::new(PermanentlyPendingReader {
6983                poll_started: Some(second_started_tx),
6984            }),
6985            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6986            lease,
6987        };
6988        let first_read_task = tokio::spawn(async move {
6989            let mut byte = [0_u8; 1];
6990            first_reader.read(&mut byte).await
6991        });
6992        let second_read_task = tokio::spawn(async move {
6993            let mut byte = [0_u8; 1];
6994            second_reader.read(&mut byte).await
6995        });
6996
6997        first_started_rx.await.expect("first inner reader should reach Poll::Pending");
6998        second_started_rx
6999            .await
7000            .expect("second inner reader should reach Poll::Pending");
7001        second_client.reject_refreshes();
7002        second_client
7003            .wait_for_rejected_refresh(Duration::from_secs(5))
7004            .await
7005            .expect("refresh rejection should be observed");
7006        let (first_result, second_result) = tokio::join!(
7007            tokio::time::timeout(Duration::from_secs(5), first_read_task),
7008            tokio::time::timeout(Duration::from_secs(5), second_read_task),
7009        );
7010        for result in [first_result, second_result] {
7011            let error = result
7012                .expect("lock loss should wake every reader whose inner I/O remains pending")
7013                .expect("reader task should not panic")
7014                .expect_err("lost snapshot lease must fail every pending read");
7015            assert_eq!(error.kind(), std::io::ErrorKind::Other);
7016        }
7017
7018        assert!(!first_client.refreshes_rejected());
7019        assert!(!first_signal.is_lost());
7020        assert!(second_signal.is_lost());
7021    }
7022
7023    #[test]
7024    fn tier_delete_entry_is_prepared_and_prefix_legacy_fails_closed() {
7025        let identity = [9_u8; 32];
7026        let mut metadata = HashMap::new();
7027        rustfs_utils::http::metadata_compat::insert_str(
7028            &mut metadata,
7029            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7030            rustfs_utils::crypto::hex(identity),
7031        );
7032        let version_id = Uuid::from_u128(1);
7033        let data_dir = Uuid::from_u128(2);
7034        let source = ObjectInfo {
7035            bucket: "bucket".to_string(),
7036            name: "object".to_string(),
7037            version_id: Some(version_id),
7038            data_dir: Some(data_dir),
7039            user_defined: Arc::new(metadata),
7040            transitioned_object: TransitionedObject {
7041                name: "remote/object".to_string(),
7042                version_id: "remote-version".to_string(),
7043                tier: "WARM".to_string(),
7044                status: TRANSITION_COMPLETE.to_string(),
7045                ..Default::default()
7046            },
7047            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
7048            ..Default::default()
7049        };
7050        let entry = build_tier_delete_journal_entry(
7051            "bucket",
7052            "object",
7053            &ObjectOptions {
7054                version_id: Some(version_id.to_string()),
7055                versioned: true,
7056                ..Default::default()
7057            },
7058            &source,
7059        )
7060        .expect("transition source should produce a journal entry")
7061        .expect("completed transition should be journaled");
7062
7063        assert_eq!(entry.state, TierDeleteJournalState::Prepared);
7064        assert_eq!(entry.backend_identity, Some(identity));
7065        let data_dir_string = data_dir.to_string();
7066        assert_eq!(
7067            entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
7068            Some(data_dir_string.as_str())
7069        );
7070
7071        let mut unknown = source.clone();
7072        unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7073        assert!(
7074            build_tier_delete_journal_entry(
7075                "bucket",
7076                "object",
7077                &ObjectOptions {
7078                    delete_prefix: true,
7079                    versioned: true,
7080                    ..Default::default()
7081                },
7082                &unknown,
7083            )
7084            .expect_err("recursive prefix delete must fail closed for legacy Unknown metadata")
7085            .to_string()
7086            .contains("legacy Unknown")
7087        );
7088
7089        assert!(
7090            build_tier_delete_journal_entry(
7091                "bucket",
7092                "object",
7093                &ObjectOptions {
7094                    delete_prefix: true,
7095                    delete_prefix_object: true,
7096                    versioned: true,
7097                    lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
7098                        version_id: Some(version_id),
7099                        delete_marker: false,
7100                        action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
7101                        rule_id: "delete-all".to_string(),
7102                        phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
7103                    }),
7104                    ..Default::default()
7105                },
7106                &unknown,
7107            )
7108            .expect("lifecycle exact-object delete-all may use its free-version fallback")
7109            .is_none()
7110        );
7111
7112        let mut identity_less = source;
7113        identity_less.user_defined = Arc::new(HashMap::new());
7114        assert!(
7115            build_tier_delete_journal_entry(
7116                "bucket",
7117                "object",
7118                &ObjectOptions {
7119                    delete_prefix: true,
7120                    versioned: true,
7121                    ..Default::default()
7122                },
7123                &identity_less,
7124            )
7125            .expect_err("recursive prefix delete must fail closed without destination identity")
7126            .to_string()
7127            .contains("requires a stable transitioned source")
7128        );
7129    }
7130
7131    #[test]
7132    fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
7133        let cancelled = Error::OperationCanceled;
7134        assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
7135        assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
7136        assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
7137        assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
7138    }
7139
7140    #[test]
7141    fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
7142        let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
7143            .expect_err("a real walk error must not be hidden by earlier cancellation");
7144        assert_eq!(err, StorageError::FileAccessDenied);
7145
7146        assert_eq!(
7147            combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
7148                .expect_err("cancellation must remain visible when there is no real error"),
7149            Error::OperationCanceled
7150        );
7151    }
7152
7153    impl Drop for BodyCacheHookGuard {
7154        fn drop(&mut self) {
7155            clear_get_object_body_cache_hook();
7156        }
7157    }
7158
7159    #[test]
7160    fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
7161        let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
7162        assert_eq!(target, None);
7163    }
7164
7165    #[test]
7166    fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
7167        let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
7168        let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
7169        assert_eq!(target, None);
7170    }
7171
7172    #[test]
7173    fn non_delete_marker_data_movement_keeps_existing_pool() {
7174        let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
7175        assert_eq!(target, Some(0));
7176    }
7177
7178    #[test]
7179    fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
7180        let version_id = Uuid::nil();
7181        let mod_time = OffsetDateTime::UNIX_EPOCH;
7182        let source = ObjectInfo {
7183            version_id: Some(version_id),
7184            delete_marker: true,
7185            mod_time: Some(mod_time),
7186            ..Default::default()
7187        };
7188        let target = source.clone();
7189
7190        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7191
7192        let mismatched = ObjectInfo {
7193            mod_time: Some(mod_time + Duration::from_secs(1)),
7194            ..target
7195        };
7196        assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
7197    }
7198
7199    #[test]
7200    fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
7201        let mut source = ObjectInfo {
7202            version_id: Some(Uuid::from_u128(1)),
7203            delete_marker: true,
7204            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7205            ..Default::default()
7206        };
7207        rustfs_utils::http::insert_str(
7208            Arc::make_mut(&mut source.user_defined),
7209            rustfs_utils::http::SUFFIX_TIER_FV_ID,
7210            Uuid::from_u128(2).to_string(),
7211        );
7212        let mut target = source.clone();
7213        rustfs_utils::http::insert_str(
7214            Arc::make_mut(&mut target.user_defined),
7215            rustfs_utils::http::SUFFIX_TIER_FV_ID,
7216            Uuid::from_u128(3).to_string(),
7217        );
7218
7219        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7220
7221        Arc::make_mut(&mut target.user_defined).insert(
7222            format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
7223            Uuid::from_u128(4).to_string(),
7224        );
7225        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7226    }
7227
7228    #[test]
7229    fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
7230        let key = format!(
7231            "{}{}",
7232            rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7233            rustfs_utils::http::SUFFIX_REPLICATION_STATUS
7234        );
7235        let timestamp_key = format!(
7236            "{}{}",
7237            rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7238            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
7239        );
7240        let source = ObjectInfo {
7241            version_id: Some(Uuid::from_u128(1)),
7242            delete_marker: true,
7243            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7244            user_defined: Arc::new(HashMap::from([
7245                (key.clone(), "arn=COMPLETED;".to_string()),
7246                (timestamp_key, "1970-01-01T00:00:01Z".to_string()),
7247            ])),
7248            ..Default::default()
7249        };
7250        let mut target = source.clone();
7251        rustfs_utils::http::insert_str(
7252            Arc::make_mut(&mut target.user_defined),
7253            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7254            "arn=COMPLETED;".to_string(),
7255        );
7256        rustfs_utils::http::insert_str(
7257            Arc::make_mut(&mut target.user_defined),
7258            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
7259            (OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
7260        );
7261        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7262
7263        Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
7264        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7265    }
7266
7267    #[test]
7268    fn equivalent_data_movement_delete_marker_ignores_target_bucket_incarnation_fence() {
7269        let source = ObjectInfo {
7270            version_id: Some(Uuid::from_u128(1)),
7271            delete_marker: true,
7272            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7273            ..Default::default()
7274        };
7275        let mut target = source.clone();
7276        rustfs_utils::http::insert_str(
7277            Arc::make_mut(&mut target.user_defined),
7278            rustfs_utils::http::SUFFIX_BUCKET_INCARNATION_ID,
7279            Uuid::from_u128(2).to_string(),
7280        );
7281
7282        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7283    }
7284
7285    #[test]
7286    fn data_movement_delete_marker_source_requires_persisted_mod_time() {
7287        let source = ObjectInfo {
7288            delete_marker: true,
7289            ..Default::default()
7290        };
7291        assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7292
7293        let source = ObjectInfo {
7294            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7295            ..source
7296        };
7297        assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
7298        assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7299    }
7300
7301    #[test]
7302    fn data_movement_delete_marker_uses_current_source_replication_state() {
7303        let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
7304        let timestamp = expected_timestamp.to_string();
7305        let mut metadata = HashMap::new();
7306        rustfs_utils::http::insert_str(
7307            &mut metadata,
7308            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7309            ReplicationStatusType::Replica.to_string(),
7310        );
7311        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
7312        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
7313        rustfs_utils::http::insert_str(
7314            &mut metadata,
7315            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7316            "arn=COMPLETED;".to_string(),
7317        );
7318        rustfs_utils::http::insert_str(
7319            &mut metadata,
7320            &format!(
7321                "{}{}",
7322                rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
7323                "arn:minio:replication::TenantA:bucket"
7324            ),
7325            "reset-id".to_string(),
7326        );
7327        rustfs_utils::http::insert_str(
7328            &mut metadata,
7329            &format!(
7330                "{}{}",
7331                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
7332                "arn:minio:replication::TenantA:bucket"
7333            ),
7334            "target-version".to_string(),
7335        );
7336        let source = ObjectInfo {
7337            delete_marker: true,
7338            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7339            replication_status_internal: Some("arn=COMPLETED;".to_string()),
7340            replication_decision: "arn=replicate;".to_string(),
7341            user_defined: Arc::new(metadata),
7342            ..Default::default()
7343        };
7344        let opts = ObjectOptions {
7345            mod_time: source.mod_time,
7346            delete_replication: Some(ReplicationState {
7347                replication_status_internal: Some("arn=PENDING;".to_string()),
7348                ..Default::default()
7349            }),
7350            ..Default::default()
7351        };
7352
7353        let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
7354        let state = target_opts.delete_replication.as_ref().expect("current replication state");
7355        assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
7356        assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
7357        assert_eq!(state.replica_timestamp, Some(expected_timestamp));
7358        assert_eq!(state.replication_timestamp, state.replica_timestamp);
7359        assert_eq!(state.replicate_decision_str, "arn=replicate;");
7360        assert_eq!(
7361            state
7362                .reset_statuses_map
7363                .get("arn:minio:replication::TenantA:bucket")
7364                .map(String::as_str),
7365            Some("reset-id")
7366        );
7367        assert_eq!(
7368            state
7369                .target_delete_marker_version_ids
7370                .get("arn:minio:replication::TenantA:bucket")
7371                .map(String::as_str),
7372            Some("target-version")
7373        );
7374    }
7375
7376    #[test]
7377    fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
7378        let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
7379        let mut malformed = HashMap::new();
7380        rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
7381        let malformed_source = ObjectInfo {
7382            delete_marker: true,
7383            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7384            user_defined: Arc::new(malformed),
7385            ..Default::default()
7386        };
7387        assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
7388
7389        let mut conflicted = HashMap::new();
7390        let suffix = format!(
7391            "{}arn:minio:replication::target:bucket",
7392            rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7393        );
7394        rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
7395        conflicted.insert(
7396            format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7397            "target-version-b".to_string(),
7398        );
7399        let conflicted_source = ObjectInfo {
7400            user_defined: Arc::new(conflicted),
7401            ..malformed_source.clone()
7402        };
7403        assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
7404
7405        let mut over_cap = HashMap::new();
7406        for index in 0..=1_000 {
7407            let suffix = format!(
7408                "{}arn:minio:replication::target:bucket-{index}",
7409                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7410            );
7411            rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
7412        }
7413        let over_cap_source = ObjectInfo {
7414            user_defined: Arc::new(over_cap),
7415            ..malformed_source
7416        };
7417        assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
7418    }
7419
7420    #[test]
7421    fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
7422        let mut source_metadata = HashMap::new();
7423        rustfs_utils::http::insert_str(
7424            &mut source_metadata,
7425            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7426            ReplicationStatusType::Replica.to_string(),
7427        );
7428        rustfs_utils::http::insert_str(
7429            &mut source_metadata,
7430            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7431            "arn=COMPLETED;".to_string(),
7432        );
7433        let source = ObjectInfo {
7434            delete_marker: true,
7435            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7436            replication_status_internal: Some("arn=COMPLETED;".to_string()),
7437            user_defined: Arc::new(source_metadata),
7438            ..Default::default()
7439        };
7440
7441        let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
7442            .expect("legacy status-only metadata should remain migratable");
7443        let state = target_opts
7444            .delete_replication
7445            .expect("replication state should be reconstructed");
7446        assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7447        assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7448
7449        let mut target_metadata = (*source.user_defined).clone();
7450        let epoch = OffsetDateTime::UNIX_EPOCH
7451            .format(&time::format_description::well_known::Rfc3339)
7452            .unwrap();
7453        rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
7454        rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
7455        assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
7456    }
7457
7458    #[test]
7459    fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
7460        let version_id = Uuid::nil();
7461        let mod_time = OffsetDateTime::UNIX_EPOCH;
7462        let source = ObjectInfo {
7463            version_id: Some(version_id),
7464            delete_marker: true,
7465            mod_time: Some(mod_time),
7466            user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
7467            replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7468            version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7469            ..Default::default()
7470        };
7471
7472        let mut target = source.clone();
7473        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
7474        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7475
7476        let mut target = source.clone();
7477        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7478        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7479
7480        let mut target = source.clone();
7481        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7482        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7483    }
7484
7485    #[test]
7486    fn equivalent_data_movement_delete_marker_rejects_live_object() {
7487        let source = ObjectInfo {
7488            delete_marker: true,
7489            ..Default::default()
7490        };
7491        let target = ObjectInfo {
7492            delete_marker: false,
7493            ..source.clone()
7494        };
7495
7496        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7497    }
7498
7499    #[test]
7500    fn data_movement_delete_marker_resume_accepts_equivalent_target() {
7501        let source = ObjectInfo {
7502            version_id: Some(Uuid::nil()),
7503            delete_marker: true,
7504            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7505            ..Default::default()
7506        };
7507
7508        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
7509            .expect("equivalent delete marker target should be evaluated");
7510
7511        assert!(should_resume);
7512    }
7513
7514    #[test]
7515    fn data_movement_delete_marker_resume_rejects_source_pool_target() {
7516        let source = ObjectInfo {
7517            version_id: Some(Uuid::nil()),
7518            delete_marker: true,
7519            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7520            ..Default::default()
7521        };
7522
7523        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
7524            .expect("source-pool target should be rejected before target lookup");
7525
7526        assert!(!should_resume);
7527    }
7528
7529    #[test]
7530    fn data_movement_resume_target_prefers_selected_non_source_pool() {
7531        let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
7532        assert_eq!(target_pool_idx, 2);
7533    }
7534
7535    #[test]
7536    fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
7537        let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
7538        assert_eq!(target_pool_idx, 3);
7539        assert!(should_check_data_movement_resume_target(1, target_pool_idx));
7540    }
7541
7542    #[test]
7543    fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
7544        let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
7545        assert_eq!(target_pool_idx, 1);
7546    }
7547
7548    #[test]
7549    fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
7550        let source = ObjectInfo {
7551            delete_marker: true,
7552            ..Default::default()
7553        };
7554        let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
7555
7556        assert!(matches!(result, Err(Error::SlowDown)));
7557    }
7558
7559    fn tiered_equivalence_source() -> FileInfo {
7560        let version_id = Uuid::nil();
7561        let transition_version_id = Uuid::new_v4();
7562        let mod_time = OffsetDateTime::UNIX_EPOCH;
7563
7564        FileInfo {
7565            version_id: Some(version_id),
7566            size: 1024,
7567            mod_time: Some(mod_time),
7568            checksum: Some(Bytes::from_static(b"checksum")),
7569            transition_status: TRANSITION_COMPLETE.to_string(),
7570            transitioned_objname: "remote/object".to_string(),
7571            transition_tier: "WARM".to_string(),
7572            transition_version_id: Some(transition_version_id),
7573            replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
7574                replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7575                targets: replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
7576                version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7577                purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
7578                ..Default::default()
7579            })),
7580            metadata: HashMap::from([
7581                ("etag".to_string(), "etag-value".to_string()),
7582                ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
7583                (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
7584                ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
7585            ]),
7586            ..Default::default()
7587        }
7588    }
7589
7590    fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
7591        ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
7592    }
7593
7594    #[test]
7595    fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
7596        let source = tiered_equivalence_source();
7597        let target = tiered_equivalence_target(&source);
7598
7599        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7600    }
7601
7602    #[test]
7603    fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
7604        let source = tiered_equivalence_source();
7605        assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
7606
7607        let mut changed_remote = source.clone();
7608        changed_remote.transitioned_objname = "remote/replaced".to_string();
7609        assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
7610
7611        let mut changed_backend = source.clone();
7612        rustfs_utils::http::metadata_compat::insert_str(
7613            &mut changed_backend.metadata,
7614            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7615            rustfs_utils::crypto::hex([9; 32]),
7616        );
7617        assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
7618    }
7619
7620    #[test]
7621    fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
7622        let mut compressed = tiered_equivalence_source();
7623        compressed.size = 600;
7624        rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
7625        rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
7626        let compressed_target = tiered_equivalence_target(&compressed);
7627        assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
7628
7629        let mut encrypted = tiered_equivalence_source();
7630        encrypted.size = 640;
7631        encrypted.metadata.insert(
7632            rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
7633            "key-id".to_string(),
7634        );
7635        encrypted.metadata.insert(
7636            rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
7637            "1024".to_string(),
7638        );
7639        let encrypted_target = tiered_equivalence_target(&encrypted);
7640        assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
7641    }
7642
7643    #[test]
7644    fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
7645        let mut source = tiered_equivalence_source();
7646        let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
7647        source.metadata.insert(
7648            format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7649            source.transition_tier.clone(),
7650        );
7651        let mut target = tiered_equivalence_target(&source);
7652        Arc::make_mut(&mut target.user_defined)
7653            .insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
7654
7655        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7656    }
7657
7658    #[test]
7659    fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
7660        let mut source = tiered_equivalence_source();
7661        source.parts = vec![rustfs_filemeta::ObjectPartInfo {
7662            number: 1,
7663            mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
7664            checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
7665            ..Default::default()
7666        }];
7667        rustfs_utils::http::insert_str(
7668            &mut source.metadata,
7669            rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
7670            r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
7671        );
7672        let mut target = tiered_equivalence_target(&source);
7673        Arc::make_mut(&mut target.parts)[0].mod_time = None;
7674        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7675
7676        let mut missing = target;
7677        Arc::make_mut(&mut missing.parts)[0].checksums = None;
7678        assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
7679    }
7680
7681    #[test]
7682    fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
7683        let source = tiered_equivalence_source();
7684        let mut target = tiered_equivalence_target(&source);
7685        target.transitioned_object.name = "remote/target".to_string();
7686
7687        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7688    }
7689
7690    #[test]
7691    fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
7692        let mut source = tiered_equivalence_source();
7693        source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
7694        let mut target = tiered_equivalence_target(&source);
7695        target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7696
7697        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7698    }
7699
7700    #[test]
7701    fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
7702        let source = tiered_equivalence_source();
7703        let mut target = tiered_equivalence_target(&source);
7704        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));
7705
7706        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7707    }
7708
7709    #[test]
7710    fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
7711        let source = tiered_equivalence_source();
7712        let mut target = tiered_equivalence_target(&source);
7713        target.user_tags = Arc::new("tag=target".to_string());
7714
7715        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7716    }
7717
7718    #[test]
7719    fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
7720        let source = tiered_equivalence_source();
7721        let mut target = tiered_equivalence_target(&source);
7722        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7723        target.replication_status = ReplicationStatusType::Failed;
7724
7725        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7726    }
7727
7728    #[test]
7729    fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
7730        let source = tiered_equivalence_source();
7731        let mut target = tiered_equivalence_target(&source);
7732        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7733        target.version_purge_status = VersionPurgeStatusType::Complete;
7734
7735        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7736    }
7737
7738    #[test]
7739    fn data_movement_tiered_resume_accepts_equivalent_target() {
7740        let source = tiered_equivalence_source();
7741        let target = tiered_equivalence_target(&source);
7742
7743        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
7744            .expect("equivalent tiered target should be evaluated");
7745
7746        assert!(should_resume);
7747    }
7748
7749    #[test]
7750    fn data_movement_tiered_resume_rejects_source_pool_target() {
7751        let source = tiered_equivalence_source();
7752        let target = tiered_equivalence_target(&source);
7753
7754        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
7755            .expect("source-pool target should be rejected before target lookup");
7756
7757        assert!(!should_resume);
7758    }
7759
7760    #[test]
7761    fn data_movement_tiered_resume_rejects_missing_target() {
7762        let source = FileInfo {
7763            version_id: Some(Uuid::nil()),
7764            size: 1024,
7765            ..Default::default()
7766        };
7767
7768        let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
7769            .expect("missing tiered target should be evaluated");
7770
7771        assert!(!should_resume);
7772    }
7773
7774    #[test]
7775    fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
7776        let info = ObjectInfo::default();
7777        let opts = ObjectOptions::default();
7778
7779        assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none());
7780    }
7781
7782    #[test]
7783    fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() {
7784        let info = ObjectInfo {
7785            delete_marker: true,
7786            ..Default::default()
7787        };
7788        let opts = ObjectOptions::default();
7789
7790        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7791            .expect("delete marker should stop latest-object reads");
7792
7793        assert!(crate::error::is_err_object_not_found(&err));
7794    }
7795
7796    #[test]
7797    fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() {
7798        let info = ObjectInfo {
7799            delete_marker: true,
7800            ..Default::default()
7801        };
7802        let opts = ObjectOptions {
7803            version_id: Some("vid-1".to_string()),
7804            ..Default::default()
7805        };
7806
7807        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7808            .expect("delete marker version reads should be rejected");
7809
7810        assert!(matches!(err, Error::MethodNotAllowed));
7811    }
7812
7813    #[test]
7814    fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() {
7815        let info = ObjectInfo {
7816            delete_marker: true,
7817            ..Default::default()
7818        };
7819        let opts = ObjectOptions {
7820            version_id: Some("vid-1".to_string()),
7821            delete_marker: true,
7822            ..Default::default()
7823        };
7824
7825        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7826            .expect("delete marker lookup should keep not-found semantics");
7827
7828        assert!(crate::error::is_err_object_not_found(&err));
7829    }
7830
7831    #[test]
7832    fn resolve_latest_object_access_returns_live_object_and_pool_idx() {
7833        let info = ObjectInfo::default();
7834        let opts = ObjectOptions::default();
7835
7836        let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap();
7837
7838        assert_eq!(idx, 7);
7839        assert!(!resolved.delete_marker);
7840    }
7841
7842    #[test]
7843    fn resolve_latest_object_access_rejects_delete_marker_without_version_id() {
7844        let info = ObjectInfo {
7845            delete_marker: true,
7846            ..Default::default()
7847        };
7848        let opts = ObjectOptions::default();
7849
7850        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7851
7852        assert!(crate::error::is_err_object_not_found(&err));
7853    }
7854
7855    #[test]
7856    fn resolve_latest_object_access_rejects_delete_marker_version_read() {
7857        let info = ObjectInfo {
7858            delete_marker: true,
7859            ..Default::default()
7860        };
7861        let opts = ObjectOptions {
7862            version_id: Some("vid-1".to_string()),
7863            ..Default::default()
7864        };
7865
7866        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7867
7868        assert!(matches!(err, Error::MethodNotAllowed));
7869    }
7870
7871    #[tokio::test]
7872    async fn latest_versioned_delete_creates_a_new_marker_after_a_current_marker() {
7873        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7874        let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7875        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7876        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7877        let bucket = "versioned-delete-marker-identity";
7878        let object = "object.bin";
7879        let opts = ObjectOptions {
7880            versioned: true,
7881            ..Default::default()
7882        };
7883
7884        store
7885            .make_bucket(bucket, &MakeBucketOptions::default())
7886            .await
7887            .expect("bucket should be created");
7888
7889        let first = store
7890            .handle_delete_object(bucket, object, opts.clone())
7891            .await
7892            .expect("first versioned delete should create a marker");
7893        let second = store
7894            .handle_delete_object(bucket, object, opts)
7895            .await
7896            .expect("second versioned delete should create a marker");
7897
7898        assert!(first.delete_marker);
7899        assert!(second.delete_marker);
7900        assert_ne!(
7901            first.version_id, second.version_id,
7902            "each latest versioned delete must persist a distinct delete marker"
7903        );
7904
7905        let versions = store.pools[0]
7906            .get_disks_by_key(object)
7907            .load_file_info_versions_exact(bucket, object)
7908            .await
7909            .expect("delete-marker metadata should decode")
7910            .expect("delete-marker metadata should be persisted");
7911        let marker_ids = versions
7912            .versions
7913            .iter()
7914            .filter(|version| version.deleted)
7915            .filter_map(|version| version.version_id)
7916            .collect::<std::collections::HashSet<_>>();
7917        assert!(marker_ids.contains(&first.version_id.expect("first marker should have a version id")));
7918        assert!(marker_ids.contains(&second.version_id.expect("second marker should have a version id")));
7919        assert_eq!(marker_ids.len(), 2, "both delete markers must remain durable versions");
7920    }
7921
7922    #[tokio::test]
7923    async fn versioned_delete_quorum_failure_rolls_back_and_retries_require_quorum() {
7924        for marker_copies in [0, 2, 4] {
7925            let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7926            let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7927            let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7928            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7929            let bucket = "versioned-delete-quorum-retry";
7930            let object = "object.bin";
7931            let opts = ObjectOptions {
7932                versioned: true,
7933                ..Default::default()
7934            };
7935            let put_opts = ObjectOptions {
7936                no_lock: true,
7937                versioned: true,
7938                ..Default::default()
7939            };
7940            store
7941                .make_bucket(bucket, &MakeBucketOptions::default())
7942                .await
7943                .expect("bucket should be created");
7944            let set = store.pools[0].get_disks_by_key(object);
7945            let payload = vec![0x5a; 4096];
7946            let mut reader = PutObjReader::from_vec(payload.clone());
7947            let original = set
7948                .put_object(bucket, object, &mut reader, &put_opts)
7949                .await
7950                .expect("the versioned 4 KiB object should be written");
7951            let disks = set.disks.read().await.clone();
7952            // Reproduce both committed markers and residue from a failed older writer.
7953            let marker = rustfs_filemeta::FileInfo {
7954                name: object.to_string(),
7955                version_id: Some(Uuid::new_v4()),
7956                deleted: true,
7957                mark_deleted: true,
7958                mod_time: Some(time::OffsetDateTime::now_utc()),
7959                ..Default::default()
7960            };
7961            for disk in disks.iter().take(marker_copies).flatten() {
7962                disk.delete_version(bucket, object, marker.clone(), true, crate::disk::DeleteOptions::default())
7963                    .await
7964                    .expect("the initial marker should be persisted on the selected disks");
7965            }
7966            let mut baseline = Vec::new();
7967            for disk in disks.iter().flatten() {
7968                baseline.push(
7969                    tokio::fs::read(disk.path().join(bucket).join(object).join("xl.meta"))
7970                        .await
7971                        .expect("initial metadata should exist"),
7972                );
7973            }
7974            {
7975                let mut online = set.disks.write().await;
7976                online[2] = None;
7977                online[3] = None;
7978            }
7979            for attempt in 1..=3 {
7980                let err = store
7981                    .handle_delete_object(bucket, object, opts.clone())
7982                    .await
7983                    .expect_err("every explicit retry must fail with only two of four disks available");
7984                assert!(
7985                    matches!(
7986                        &err,
7987                        StorageError::InsufficientWriteQuorum(error_bucket, error_object)
7988                            if error_bucket == bucket && error_object == object
7989                    ),
7990                    "expected write quorum failure with {marker_copies} marker copies, attempt {attempt}: {err:?}"
7991                );
7992                for (index, disk) in disks.iter().enumerate() {
7993                    let metadata_path = disk
7994                        .as_ref()
7995                        .expect("saved disk should exist")
7996                        .path()
7997                        .join(bucket)
7998                        .join(object);
7999                    let actual = tokio::fs::read(metadata_path.join("xl.meta"))
8000                        .await
8001                        .expect("rollback must preserve readable metadata");
8002                    assert_eq!(
8003                        actual, baseline[index],
8004                        "failed DELETE must restore exact metadata on disk {index}, attempt {attempt}"
8005                    );
8006                    let mut entries = tokio::fs::read_dir(&metadata_path)
8007                        .await
8008                        .expect("the object directory should remain readable");
8009                    while let Some(entry) = entries.next_entry().await.expect("directory entries should be readable") {
8010                        assert!(
8011                            entry.file_name() != "xl.meta.bkp" && !entry.path().join("xl.meta.bkp").exists(),
8012                            "rollback must consume staged metadata backups on disk {index}"
8013                        );
8014                    }
8015                }
8016            }
8017            *set.disks.write().await = disks;
8018            if marker_copies != 0 {
8019                continue;
8020            }
8021            let committed = store
8022                .handle_delete_object(bucket, object, opts)
8023                .await
8024                .expect("DELETE should succeed after write quorum recovers");
8025            assert!(committed.delete_marker);
8026            assert_ne!(committed.version_id, marker.version_id);
8027            let mut restored = Vec::new();
8028            set.get_object_reader(
8029                bucket,
8030                object,
8031                None,
8032                HeaderMap::new(),
8033                &ObjectOptions {
8034                    version_id: original.version_id.map(|id| id.to_string()),
8035                    ..Default::default()
8036                },
8037            )
8038            .await
8039            .expect("the original version must remain readable")
8040            .stream
8041            .read_to_end(&mut restored)
8042            .await
8043            .expect("the original version should stream");
8044            assert_eq!(restored, payload);
8045        }
8046    }
8047
8048    #[test]
8049    fn latest_versioned_delete_marker_creation_excludes_specialized_deletes() {
8050        assert!(latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8051            versioned: true,
8052            ..Default::default()
8053        }));
8054        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8055            versioned: true,
8056            version_id: Some(Uuid::new_v4().to_string()),
8057            ..Default::default()
8058        }));
8059        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8060            versioned: true,
8061            delete_marker: true,
8062            ..Default::default()
8063        }));
8064        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8065            versioned: true,
8066            data_movement: true,
8067            ..Default::default()
8068        }));
8069        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8070            versioned: true,
8071            replication_request: true,
8072            ..Default::default()
8073        }));
8074        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8075            versioned: true,
8076            version_suspended: true,
8077            ..Default::default()
8078        }));
8079    }
8080
8081    #[test]
8082    fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
8083        let opts = ObjectOptions {
8084            versioned: true,
8085            ..Default::default()
8086        };
8087
8088        assert!(should_create_delete_marker_for_missing_object(&opts));
8089    }
8090
8091    #[test]
8092    fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
8093        let version_delete = ObjectOptions {
8094            versioned: true,
8095            version_id: Some("vid-1".to_string()),
8096            ..Default::default()
8097        };
8098        let delete_marker_replication = ObjectOptions {
8099            versioned: true,
8100            delete_marker: true,
8101            ..Default::default()
8102        };
8103        let data_movement = ObjectOptions {
8104            versioned: true,
8105            data_movement: true,
8106            ..Default::default()
8107        };
8108
8109        assert!(!should_create_delete_marker_for_missing_object(&version_delete));
8110        assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
8111        assert!(!should_create_delete_marker_for_missing_object(&data_movement));
8112    }
8113
8114    // issue #4189 regression: `del_opts` pins `version_id = Uuid::nil()` on directory
8115    // keys, so deleting a ghost folder over HTTP fails the lookup with *version*-not-found
8116    // (not object-not-found). The orphan-purge guard must accept both misses, or the
8117    // ghost tree survives behind a fake 204 — the exact reported symptom.
8118    #[test]
8119    fn should_purge_orphan_dir_on_version_not_found_for_dir_key() {
8120        assert!(
8121            should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "ghost/"),
8122            "the real HTTP delete path yields version-not-found on dir keys and must reach the purge"
8123        );
8124        assert!(
8125            should_purge_orphan_dir_on_missing(
8126                &StorageError::VersionNotFound("bucket".into(), "ghost/".into(), Uuid::nil().to_string()),
8127                "ghost/"
8128            ),
8129            "typed VersionNotFound on a dir key must also reach the purge"
8130        );
8131    }
8132
8133    #[test]
8134    fn should_purge_orphan_dir_on_object_not_found_for_dir_key() {
8135        assert!(should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "ghost/"));
8136        assert!(should_purge_orphan_dir_on_missing(
8137            &StorageError::ObjectNotFound("bucket".into(), "ghost/".into()),
8138            "ghost/"
8139        ));
8140    }
8141
8142    #[test]
8143    fn should_not_purge_orphan_dir_for_regular_key_or_other_errors() {
8144        // A regular (non-directory) key must never trigger a prefix purge, even on a miss.
8145        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "regular.txt"));
8146        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "regular.txt"));
8147        // Non-miss errors (e.g. quorum failures) must not be masked by a purge attempt.
8148        assert!(!should_purge_orphan_dir_on_missing(&StorageError::ErasureReadQuorum, "ghost/"));
8149    }
8150
8151    #[test]
8152    fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
8153        let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();
8154
8155        assert_eq!(idx, 3);
8156    }
8157
8158    #[test]
8159    fn resolve_decommission_target_pool_idx_result_wraps_error_context() {
8160        let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object")
8161            .expect_err("expected contextual error");
8162        let rendered = err.to_string();
8163
8164        assert!(rendered.contains("failed to select decommission target pool"), "{rendered}");
8165        assert!(rendered.contains("bucket"), "{rendered}");
8166        assert!(rendered.contains("object"), "{rendered}");
8167        assert!(rendered.contains("boom"), "{rendered}");
8168    }
8169
8170    #[test]
8171    fn resolve_decommission_tiered_object_result_passthrough_ok() {
8172        ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object")
8173            .expect("successful decommission result should pass through");
8174    }
8175
8176    #[test]
8177    fn resolve_decommission_tiered_object_result_wraps_error_context() {
8178        let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object")
8179            .expect_err("expected contextual error");
8180        let rendered = err.to_string();
8181
8182        assert!(rendered.contains("failed to decommission tiered object"), "{rendered}");
8183        assert!(rendered.contains("bucket"), "{rendered}");
8184        assert!(rendered.contains("object"), "{rendered}");
8185        assert!(rendered.contains("boom"), "{rendered}");
8186    }
8187
8188    #[test]
8189    fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() {
8190        let err = ECStore::resolve_decommission_tiered_object_result(
8191            Err(Error::DecommissionCapacityBlocked {
8192                message: "target gate busy".to_string(),
8193            }),
8194            "bucket",
8195            "object",
8196        )
8197        .expect_err("expected contextual error");
8198
8199        assert!(matches!(
8200            crate::data_movement::data_movement_stage_source(&err),
8201            Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy"
8202        ));
8203    }
8204
8205    #[test]
8206    fn version_aware_lookup_opts_enables_version_aware_lookup() {
8207        let opts = ObjectOptions {
8208            version_id: Some("vid-1".to_string()),
8209            ..Default::default()
8210        };
8211
8212        let lookup_opts = version_aware_lookup_opts(&opts, true);
8213
8214        assert!(lookup_opts.no_lock);
8215        assert!(lookup_opts.metadata_chg);
8216        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8217    }
8218
8219    #[test]
8220    fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() {
8221        let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true);
8222
8223        assert!(lookup_opts.no_lock);
8224        assert!(!lookup_opts.metadata_chg);
8225        assert!(lookup_opts.version_id.is_none());
8226    }
8227
8228    #[test]
8229    fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() {
8230        let opts = ObjectOptions {
8231            version_id: Some("vid-1".to_string()),
8232            ..Default::default()
8233        };
8234
8235        let lookup_opts = data_movement_pool_lookup_opts(&opts, false);
8236
8237        assert!(!lookup_opts.no_lock);
8238        assert!(lookup_opts.metadata_chg);
8239        assert!(lookup_opts.skip_decommissioned);
8240        assert!(lookup_opts.skip_rebalancing);
8241        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8242    }
8243
8244    #[test]
8245    fn writer_pool_lookup_opts_skips_rebalance_sources() {
8246        let lookup_opts = writer_pool_lookup_opts(
8247            &ObjectOptions {
8248                version_id: Some("vid-1".to_string()),
8249                ..Default::default()
8250            },
8251            true,
8252        );
8253
8254        assert!(lookup_opts.no_lock);
8255        assert!(lookup_opts.metadata_chg);
8256        assert!(lookup_opts.skip_decommissioned);
8257        assert!(lookup_opts.skip_rebalancing);
8258        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8259    }
8260
8261    #[test]
8262    fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() {
8263        let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true);
8264
8265        assert!(lookup_opts.no_lock);
8266        assert!(!lookup_opts.skip_decommissioned);
8267        assert!(lookup_opts.skip_rebalancing);
8268
8269        let explicit_version = delete_pool_lookup_opts(
8270            &ObjectOptions {
8271                versioned: true,
8272                version_id: Some(uuid::Uuid::new_v4().to_string()),
8273                ..Default::default()
8274            },
8275            true,
8276        );
8277        assert!(!explicit_version.skip_decommissioned);
8278    }
8279
8280    #[test]
8281    fn delete_fans_out_for_unversioned_and_explicit_version_mutations() {
8282        assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1));
8283        assert!(should_delete_from_all_pools(
8284            &ObjectOptions {
8285                versioned: true,
8286                version_id: Some(uuid::Uuid::new_v4().to_string()),
8287                ..Default::default()
8288            },
8289            2,
8290        ));
8291        assert!(!should_delete_from_all_pools(
8292            &ObjectOptions {
8293                versioned: true,
8294                ..Default::default()
8295            },
8296            1,
8297        ));
8298        assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0));
8299    }
8300
8301    #[test]
8302    fn batch_delete_identifies_only_latest_versioned_markers() {
8303        let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test(
8304            s3s::dto::VersioningConfiguration {
8305                status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8306                ..Default::default()
8307            },
8308            None,
8309        );
8310        let latest = ObjectToDelete {
8311            object_name: "latest".to_string(),
8312            ..Default::default()
8313        };
8314        assert!(batch_delete_creates_latest_marker(&latest, &versioned));
8315        assert!(!batch_delete_targets_pool(true, Some(1), 0));
8316        assert!(batch_delete_targets_pool(true, Some(1), 1));
8317        assert!(!batch_delete_targets_pool(true, Some(1), 2));
8318
8319        let explicit = ObjectToDelete {
8320            object_name: "explicit".to_string(),
8321            version_id: Some(uuid::Uuid::new_v4()),
8322            ..Default::default()
8323        };
8324        assert!(!batch_delete_creates_latest_marker(&explicit, &versioned));
8325        assert!(batch_delete_targets_pool(false, Some(1), 0));
8326
8327        let unversioned = DeleteReplicationConfigSnapshot::default();
8328        assert!(!batch_delete_creates_latest_marker(&latest, &unversioned));
8329        assert!(batch_delete_targets_pool(false, None, 0));
8330    }
8331
8332    #[test]
8333    fn batch_delete_pool_failures_override_success_in_any_pool_order() {
8334        let success = DeletedObject {
8335            object_name: "object".to_string(),
8336            found: true,
8337            ..Default::default()
8338        };
8339        let source_errors = [
8340            StorageError::ErasureWriteQuorum,
8341            StorageError::NamespaceLockQuorumUnavailable {
8342                mode: "delete_objects_commit",
8343                bucket: "bucket".to_string(),
8344                object: "object".to_string(),
8345                required: 1,
8346                achieved: 0,
8347            },
8348        ];
8349
8350        for source_error in source_errors {
8351            for source_first in [true, false] {
8352                let failed = (DeletedObject::default(), Some(source_error.clone()));
8353                let succeeded = (success.clone(), None);
8354                let pool_results = if source_first {
8355                    vec![failed, succeeded]
8356                } else {
8357                    vec![succeeded, failed]
8358                };
8359
8360                let (_, error, attempted) =
8361                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8362
8363                assert!(attempted);
8364                assert_eq!(error, Some(source_error.clone()));
8365            }
8366        }
8367    }
8368
8369    #[test]
8370    fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() {
8371        let success = DeletedObject {
8372            object_name: "object".to_string(),
8373            found: true,
8374            ..Default::default()
8375        };
8376        let missing_errors = [
8377            StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
8378            StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
8379        ];
8380
8381        for missing_error in missing_errors {
8382            let missing = (DeletedObject::default(), Some(missing_error.clone()));
8383            for missing_first in [true, false] {
8384                let succeeded = (success.clone(), None);
8385                let pool_results = if missing_first {
8386                    vec![missing.clone(), succeeded]
8387                } else {
8388                    vec![succeeded, missing.clone()]
8389                };
8390                let (deleted, error, attempted) =
8391                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8392
8393                assert!(attempted);
8394                let deleted = deleted.expect("successful pool result should be retained");
8395                assert!(deleted.found);
8396                assert_eq!(deleted.object_name, success.object_name.as_str());
8397                assert!(error.is_none());
8398            }
8399
8400            let missing_only = [missing];
8401            let (_, error, attempted) =
8402                resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error)));
8403            assert!(attempted);
8404            assert_eq!(error, Some(missing_error));
8405        }
8406
8407        let silent_missing = [(DeletedObject::default(), None)];
8408        let (_, error, attempted) =
8409            resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error)));
8410        assert!(attempted);
8411        assert!(error.is_none());
8412    }
8413
8414    #[tokio::test]
8415    async fn lifecycle_null_version_cas_distinguishes_absent_and_replacement_pools() {
8416        use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
8417
8418        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8419        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8420        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8421        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
8422        let bucket = RUSTFS_META_BUCKET;
8423        let suspended_snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
8424            VersioningConfiguration {
8425                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
8426                ..Default::default()
8427            },
8428            None,
8429        ));
8430        let write_opts = ObjectOptions {
8431            no_lock: true,
8432            version_suspended: true,
8433            ..Default::default()
8434        };
8435        let delete_opts = ObjectOptions {
8436            version_suspended: true,
8437            delete_replication_config_snapshot: Some(Arc::clone(&suspended_snapshot)),
8438            ..Default::default()
8439        };
8440
8441        let success_object = "lifecycle-null-only-in-first-pool";
8442        let mut success_reader = PutObjReader::from_vec(b"single-pool null generation".to_vec());
8443        let success_source = first_set
8444            .put_object(bucket, success_object, &mut success_reader, &write_opts)
8445            .await
8446            .expect("the historical null generation should be written to one pool");
8447        let success_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&success_source)
8448            .expect("the null generation should produce an exact lifecycle target");
8449
8450        let (deleted, errors) = store
8451            .handle_delete_objects(bucket, vec![success_target.clone()], delete_opts.clone())
8452            .await;
8453        assert!(
8454            errors.iter().all(Option::is_none),
8455            "an absent peer pool must not override success: {errors:?}"
8456        );
8457        assert!(deleted[0].found, "the pool containing the exact null generation must report success");
8458
8459        let (deleted, errors) = store
8460            .handle_delete_objects(bucket, vec![success_target], delete_opts.clone())
8461            .await;
8462        assert!(
8463            errors.iter().all(Option::is_none),
8464            "an exact lifecycle delete retry must be idempotent: {errors:?}"
8465        );
8466        assert!(!deleted[0].found, "the retried generation should already be absent from every pool");
8467
8468        let mismatch_object = "lifecycle-null-replaced-in-second-pool";
8469        let mut original_reader = PutObjReader::from_vec(b"original null generation".to_vec());
8470        let original = first_set
8471            .put_object(bucket, mismatch_object, &mut original_reader, &write_opts)
8472            .await
8473            .expect("the original null generation should be written to the first pool");
8474        let stale_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&original)
8475            .expect("the original null generation should produce an exact lifecycle target");
8476        let mut replacement_reader = PutObjReader::from_vec(b"replacement null generation".to_vec());
8477        let replacement = second_set
8478            .put_object(bucket, mismatch_object, &mut replacement_reader, &write_opts)
8479            .await
8480            .expect("the replacement null generation should be written to the second pool");
8481        assert_ne!(original.data_dir, replacement.data_dir);
8482
8483        let (_deleted, errors) = store.handle_delete_objects(bucket, vec![stale_target], delete_opts).await;
8484        assert!(
8485            matches!(errors.as_slice(), [Some(StorageError::PreconditionFailed)]),
8486            "a different null generation in any pool must fail the aggregate CAS: {errors:?}"
8487        );
8488        let retained = second_set
8489            .get_object_info(
8490                bucket,
8491                mismatch_object,
8492                &ObjectOptions {
8493                    version_id: Some(Uuid::nil().to_string()),
8494                    version_suspended: true,
8495                    no_lock: true,
8496                    ..Default::default()
8497                },
8498            )
8499            .await
8500            .expect("the replacement null generation must remain after a stale CAS");
8501        assert_eq!(retained.data_dir, replacement.data_dir);
8502    }
8503
8504    #[test]
8505    fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
8506        let lookup_opts = data_movement_pool_lookup_opts(
8507            &ObjectOptions {
8508                version_id: Some("vid-1".to_string()),
8509                ..Default::default()
8510            },
8511            true,
8512        );
8513
8514        assert!(lookup_opts.no_lock);
8515        assert!(lookup_opts.metadata_chg);
8516        assert!(lookup_opts.skip_decommissioned);
8517        assert!(lookup_opts.skip_rebalancing);
8518    }
8519
8520    #[test]
8521    fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() {
8522        let mut opts = ObjectOptions {
8523            data_movement: true,
8524            ..Default::default()
8525        };
8526        assert!(!uses_data_movement_pool_selection(&opts));
8527
8528        DecommissionCapacityOwner {
8529            source_pool_index: 1,
8530            operation_id: Uuid::new_v4(),
8531            generation: 2,
8532            owner_nonce: Uuid::new_v4(),
8533            mutation_id: None,
8534        }
8535        .apply_to(&mut opts);
8536
8537        assert!(uses_data_movement_pool_selection(&opts));
8538    }
8539
8540    #[test]
8541    fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
8542        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8543            no_lock: false,
8544            skip_decommissioned: false,
8545            ..Default::default()
8546        });
8547
8548        assert!(lookup_opts.skip_decommissioned);
8549        assert!(lookup_opts.skip_rebalancing);
8550        assert!(!lookup_opts.no_lock);
8551    }
8552
8553    #[test]
8554    fn transition_restore_pool_opts_rejects_ambient_no_lock() {
8555        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8556            no_lock: true,
8557            ..Default::default()
8558        });
8559
8560        assert!(lookup_opts.skip_decommissioned);
8561        assert!(lookup_opts.skip_rebalancing);
8562        assert!(!lookup_opts.no_lock);
8563    }
8564
8565    #[test]
8566    fn delete_objects_lock_names_are_sorted_and_unique() {
8567        let objects = vec![
8568            ObjectToDelete {
8569                object_name: "beta".to_string(),
8570                ..Default::default()
8571            },
8572            ObjectToDelete {
8573                object_name: "alpha".to_string(),
8574                ..Default::default()
8575            },
8576            ObjectToDelete {
8577                object_name: "beta".to_string(),
8578                ..Default::default()
8579            },
8580        ];
8581
8582        assert_eq!(sorted_unique_delete_object_names(&objects), vec!["alpha", "beta"]);
8583    }
8584
8585    async fn new_read_lock_test_store() -> ECStore {
8586        let format = FormatV3::new(1, 2);
8587        let endpoints = vec![
8588            Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
8589            Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
8590        ];
8591        let pool_endpoints = PoolEndpoints {
8592            legacy: false,
8593            set_count: 1,
8594            drives_per_set: 2,
8595            endpoints: Endpoints::from(endpoints),
8596            cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
8597            platform: "test".to_string(),
8598        };
8599        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
8600        let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
8601            .await
8602            .expect("test sets should be created with empty disks");
8603
8604        ECStore {
8605            id: Uuid::new_v4(),
8606            disk_map: HashMap::new(),
8607            pools: vec![sets],
8608            peer_sys: S3PeerSys::new(&endpoint_pools),
8609            pool_meta: RwLock::new(PoolMeta::default()),
8610            rebalance_meta: RwLock::new(None),
8611            decommission_cancelers: RwLock::new(Vec::new()),
8612            start_gate: Mutex::new(()),
8613            pool_meta_save_gate: Mutex::default(),
8614            ctx: crate::runtime::instance::bootstrap_ctx(),
8615            bucket_fence_registry: std::sync::Arc::default(),
8616        }
8617    }
8618
8619    #[tokio::test]
8620    async fn delete_object_handler_futures_remain_stack_bounded() {
8621        let store = new_read_lock_test_store().await;
8622        let unified_future = store.handle_delete_object_with_journal("bucket", "object", ObjectOptions::default(), None);
8623        let unified_future_size = std::mem::size_of_val(&unified_future);
8624
8625        assert!(
8626            unified_future_size <= 4 * 1024,
8627            "unified delete handler future must remain stack-bounded; measured {unified_future_size} bytes"
8628        );
8629        drop(unified_future);
8630
8631        let batch_future =
8632            store.handle_delete_objects_with_journal_and_accounting("bucket", Vec::new(), ObjectOptions::default(), None);
8633        let batch_future_size = std::mem::size_of_val(&batch_future);
8634        assert!(
8635            batch_future_size <= 4 * 1024,
8636            "batch delete handler future must remain stack-bounded; measured {batch_future_size} bytes"
8637        );
8638        drop(batch_future);
8639
8640        let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default());
8641        let outer_future_size = std::mem::size_of_val(&outer_future);
8642        assert!(
8643            outer_future_size <= 16 * 1024,
8644            "outer delete handler future must remain stack-bounded; measured {outer_future_size} bytes"
8645        );
8646    }
8647
8648    async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
8649        new_prepared_reader_test_store_with_ctx(set_disks, crate::runtime::instance::bootstrap_ctx()).await
8650    }
8651
8652    async fn new_prepared_reader_test_store_with_ctx(
8653        set_disks: &[Arc<SetDisks>],
8654        ctx: Arc<crate::runtime::instance::InstanceContext>,
8655    ) -> ECStore {
8656        let mut pool_configs = Vec::with_capacity(set_disks.len());
8657        let mut pools = Vec::with_capacity(set_disks.len());
8658
8659        for (pool_idx, set_disks) in set_disks.iter().enumerate() {
8660            let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
8661            for endpoint in endpoints.as_mut() {
8662                endpoint.set_pool_index(pool_idx);
8663            }
8664            let pool_config = PoolEndpoints {
8665                legacy: false,
8666                set_count: 1,
8667                drives_per_set: set_disks.set_drive_count,
8668                endpoints,
8669                cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
8670                platform: "test".to_string(),
8671            };
8672            let disks = set_disks.disks.read().await.clone();
8673            let pool = Sets::new_with_instance_ctx(
8674                disks,
8675                &pool_config,
8676                &set_disks.format,
8677                pool_idx,
8678                set_disks.default_parity_count,
8679                Arc::clone(&ctx),
8680            )
8681            .await
8682            .expect("prepared-reader test pool should be created from local disks");
8683            pool_configs.push(pool_config);
8684            pools.push(pool);
8685        }
8686
8687        new_prepared_reader_test_store_from_pools(pools, pool_configs, ctx)
8688    }
8689
8690    fn new_prepared_reader_test_store_from_pools(
8691        pools: Vec<Arc<Sets>>,
8692        pool_configs: Vec<PoolEndpoints>,
8693        ctx: Arc<crate::runtime::instance::InstanceContext>,
8694    ) -> ECStore {
8695        let endpoint_pools = EndpointServerPools::from(pool_configs);
8696        ECStore {
8697            id: Uuid::new_v4(),
8698            disk_map: HashMap::new(),
8699            pools,
8700            peer_sys: S3PeerSys::new_with_instance_ctx(&endpoint_pools, Arc::clone(&ctx)),
8701            pool_meta: RwLock::new(PoolMeta::default()),
8702            rebalance_meta: RwLock::new(None),
8703            decommission_cancelers: RwLock::new(Vec::new()),
8704            start_gate: Mutex::new(()),
8705            pool_meta_save_gate: Mutex::default(),
8706            ctx,
8707            bucket_fence_registry: std::sync::Arc::default(),
8708        }
8709    }
8710
8711    #[tokio::test]
8712    #[serial_test::serial]
8713    async fn object_lock_diagnostics_report_historical_get_blocked_by_put_commit() {
8714        use crate::diagnostics::object_lock::tests::CapturedLockEvents;
8715        use crate::object_api::WriteCompletion;
8716        use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
8717        use tracing::{Instrument, instrument::WithSubscriber};
8718
8719        let capture = CapturedLockEvents::default();
8720        temp_env::async_with_vars(
8721            [
8722                (rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true")),
8723                (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
8724            ],
8725            async {
8726                let bucket = "historical-get-lock-diagnostics";
8727                let object = "object.bin";
8728                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8729                let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, ctx.clone()).await;
8730                let store = new_prepared_reader_test_store_with_ctx(&[original], ctx).await;
8731                let set = store.pools[0].get_disks_by_key(object);
8732                set.make_bucket(bucket, &MakeBucketOptions::default())
8733                    .await
8734                    .expect("create bucket");
8735                let opts = ObjectOptions {
8736                    versioned: true,
8737                    write_completion: WriteCompletion::TailDrained,
8738                    ..Default::default()
8739                };
8740                let mut versions = Vec::new();
8741                for value in [0x11, 0x22, 0x33] {
8742                    let info = set
8743                        .put_object(bucket, object, &mut PutObjReader::from_vec(vec![value; 8193]), &opts)
8744                        .await
8745                        .expect("seed version");
8746                    versions.push((info.version_id.expect("explicit seed version"), value));
8747                }
8748
8749                let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
8750                let put_set = set.clone();
8751                let put = tokio::spawn(
8752                    async move {
8753                        put_set
8754                            .put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x44; 8193]), &opts)
8755                            .await
8756                    }
8757                    .instrument(tracing::info_span!("request", request_id = "diagnostic-put"))
8758                    .with_current_subscriber(),
8759                );
8760                barrier.wait_until_paused().await;
8761
8762                let historical_version = versions[0].0.to_string();
8763                let result = store
8764                    .prepare_get_object_reader(
8765                        bucket,
8766                        object,
8767                        None,
8768                        HeaderMap::new(),
8769                        &ObjectOptions {
8770                            version_id: Some(historical_version.clone()),
8771                            versioned: true,
8772                            ..Default::default()
8773                        },
8774                    )
8775                    .instrument(tracing::info_span!("request", request_id = "diagnostic-get"))
8776                    .await;
8777                assert!(
8778                    matches!(result, Err(StorageError::Lock(rustfs_lock::LockError::Timeout { .. }))),
8779                    "a held PUT commit must preserve the typed GET lock timeout"
8780                );
8781                let rows = capture.rows();
8782                let failure = rows
8783                    .iter()
8784                    .find(|row| row["fields"]["op"] == "prepare_get_object" && row["fields"]["state"] == "failed")
8785                    .expect("failed GET acquisition must be observable before error propagation");
8786                assert_eq!(failure["fields"]["requested_version_id"], historical_version);
8787                assert_eq!(failure["fields"]["failure"], "timeout");
8788                assert_eq!(failure["fields"]["timeout_ms"], 1000);
8789                assert!(failure["fields"]["acquire_ms"].as_u64().expect("wait duration") >= 1000);
8790                assert!(
8791                    failure["spans"]
8792                        .as_array()
8793                        .expect("request spans")
8794                        .iter()
8795                        .any(|span| span["request_id"] == "diagnostic-get")
8796                );
8797                let put_acquired = rows
8798                    .iter()
8799                    .rev()
8800                    .find(|row| row["fields"]["op"] == "put_object_commit" && row["fields"]["state"] == "acquired")
8801                    .expect("PUT lock holder event");
8802                assert_ne!(put_acquired["fields"]["lock_attempt_id"], failure["fields"]["lock_attempt_id"]);
8803                assert!(
8804                    !rows
8805                        .iter()
8806                        .any(|row| row["fields"]["lock_attempt_id"] == put_acquired["fields"]["lock_attempt_id"]
8807                            && row["fields"]["state"] == "guard_dropped"),
8808                    "a paused commit still owns the lock"
8809                );
8810
8811                barrier.release();
8812                let new_version = put
8813                    .await
8814                    .expect("PUT task")
8815                    .expect("new version commits")
8816                    .version_id
8817                    .expect("new version id");
8818                versions.push((new_version, 0x44));
8819                for (version_id, value) in versions {
8820                    let prepared = store
8821                        .prepare_get_object_reader(
8822                            bucket,
8823                            object,
8824                            None,
8825                            HeaderMap::new(),
8826                            &ObjectOptions {
8827                                version_id: Some(version_id.to_string()),
8828                                versioned: true,
8829                                ..Default::default()
8830                            },
8831                        )
8832                        .await
8833                        .expect("every acknowledged version remains readable");
8834                    assert_eq!(prepared.object_info().version_id, Some(version_id));
8835                    let mut reader = prepared.into_reader().await.expect("open historical body");
8836                    let mut body = Vec::new();
8837                    reader
8838                        .stream
8839                        .read_to_end(&mut body)
8840                        .await
8841                        .expect("read complete version body");
8842                    assert_eq!(body, vec![value; 8193]);
8843                }
8844            }
8845            .with_subscriber(capture.subscriber()),
8846        )
8847        .await;
8848    }
8849
8850    async fn multipool_version_test_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
8851        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8852        let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8853        let (second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8854        dirs.extend(second_dirs);
8855        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first_set, second_set], ctx).await);
8856        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
8857        store
8858            .handle_make_bucket(
8859                bucket,
8860                &MakeBucketOptions {
8861                    versioning_enabled: true,
8862                    ..Default::default()
8863                },
8864            )
8865            .await
8866            .expect("create the versioned bucket in both pools");
8867        (dirs, store)
8868    }
8869
8870    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8871    async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
8872        let bucket = "retired-marker-c11";
8873        let object = "marker.bin";
8874        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8875        let (dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
8876        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
8877        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8878        let create = MakeBucketOptions {
8879            versioning_enabled: true,
8880            ..Default::default()
8881        };
8882        store.handle_make_bucket(bucket, &create).await.expect("create old bucket");
8883        let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old identity");
8884        let mut historical = Vec::new();
8885        for value in 1..=3 {
8886            historical.push(
8887                store
8888                    .put_object(
8889                        bucket,
8890                        "history.bin",
8891                        &mut PutObjReader::from_vec(vec![value; 4097]),
8892                        &ObjectOptions {
8893                            versioned: true,
8894                            ..Default::default()
8895                        },
8896                    )
8897                    .await
8898                    .expect("write old version")
8899                    .version_id
8900                    .unwrap(),
8901            );
8902        }
8903        let marker = store
8904            .delete_object(
8905                bucket,
8906                object,
8907                ObjectOptions {
8908                    versioned: true,
8909                    ..Default::default()
8910                },
8911            )
8912            .await
8913            .expect("create old marker")
8914            .version_id
8915            .expect("explicit marker version");
8916        assert!(
8917            store
8918                .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8919                .await
8920                .is_err()
8921        );
8922        assert!(
8923            !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8924                .await
8925                .unwrap(),
8926            "failed bucket deletion cannot publish retirement"
8927        );
8928        let set = &store.pools[0].disk_set[0];
8929        let before = set.disks.read().await.clone();
8930        for disk in before.iter().flatten() {
8931            let info = disk
8932                .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
8933                .await
8934                .expect("read original marker");
8935            assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation));
8936        }
8937        // Taking four actual members offline preserves their existing xl.meta.
8938        // All deletion/recreation work below goes through the normal store API.
8939        for slot in 12..16 {
8940            set.disks.write().await[slot] = None;
8941        }
8942        for version in historical {
8943            store
8944                .delete_object(
8945                    bucket,
8946                    "history.bin",
8947                    ObjectOptions {
8948                        versioned: true,
8949                        version_id: Some(version.to_string()),
8950                        ..Default::default()
8951                    },
8952                )
8953                .await
8954                .expect("delete exact old data version with four drives offline");
8955        }
8956        store
8957            .delete_object(
8958                bucket,
8959                object,
8960                ObjectOptions {
8961                    versioned: true,
8962                    version_id: Some(marker.to_string()),
8963                    ..Default::default()
8964                },
8965            )
8966            .await
8967            .expect("delete exact old marker with four drives offline");
8968        store
8969            .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8970            .await
8971            .expect("delete empty old bucket");
8972        assert!(
8973            crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8974                .await
8975                .expect("durable retirement")
8976        );
8977        store
8978            .handle_make_bucket(bucket, &create)
8979            .await
8980            .expect("recreate same bucket name");
8981        let current = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
8982        assert_ne!(old_incarnation, current);
8983        let mut new_versions = Vec::new();
8984        for (key, payload) in [
8985            (object, b"new generation reused key".as_slice()),
8986            ("new.bin", b"new generation control".as_slice()),
8987        ] {
8988            let info = store
8989                .put_object(
8990                    bucket,
8991                    key,
8992                    &mut PutObjReader::from_vec(payload.to_vec()),
8993                    &ObjectOptions {
8994                        versioned: true,
8995                        ..Default::default()
8996                    },
8997                )
8998                .await
8999                .expect("write new generation");
9000            new_versions.push((key, info.version_id.unwrap(), payload));
9001        }
9002        *set.disks.write().await = before.clone();
9003        let heal = rustfs_heal_contracts::heal_channel::HealOpts {
9004            remove: true,
9005            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9006            ..Default::default()
9007        };
9008        let observed = store
9009            .clone()
9010            .inner_list_object_versions(bucket, "", None, None, None, 100)
9011            .await
9012            .expect("list after rejoin");
9013        assert_eq!(observed.objects.len(), 2, "stale marker must not contaminate listing");
9014        assert!(observed.objects.iter().all(|info| !info.delete_marker));
9015        for (key, version, _) in &new_versions {
9016            let repaired = store
9017                .heal_object_with_proof(bucket, key, &version.to_string(), &heal)
9018                .await
9019                .expect("heal new generation");
9020            assert!(repaired.error.is_none(), "new version heal: {:?}", repaired.error);
9021        }
9022        let no_remove = store
9023            .heal_object_with_proof(
9024                bucket,
9025                object,
9026                &marker.to_string(),
9027                &rustfs_heal_contracts::heal_channel::HealOpts { remove: false, ..heal },
9028            )
9029            .await
9030            .expect("inspect marker without removal");
9031        assert!(no_remove.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9032        assert!(no_remove.absence.is_none());
9033        let dry = store
9034            .heal_object_with_proof(
9035                bucket,
9036                object,
9037                &marker.to_string(),
9038                &rustfs_heal_contracts::heal_channel::HealOpts { dry_run: true, ..heal },
9039            )
9040            .await
9041            .expect("dry run");
9042        assert!(dry.absence.is_none());
9043        for disk in before.iter().skip(12).flatten() {
9044            assert!(
9045                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9046                    .await
9047                    .is_ok()
9048            );
9049        }
9050        let original_marker = before[12]
9051            .as_ref()
9052            .unwrap()
9053            .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9054            .await
9055            .unwrap();
9056        for stamp in [None, Some(current), Some(Uuid::new_v4())] {
9057            for disk in before.iter().skip(12).flatten() {
9058                let mut changed = original_marker.clone();
9059                changed.metadata.clear();
9060                if let Some(stamp) = stamp {
9061                    changed.set_delete_marker_incarnation(stamp);
9062                }
9063                disk.write_metadata("", bucket, object, changed).await.unwrap();
9064            }
9065            let unresolved = store
9066                .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9067                .await
9068                .unwrap();
9069            assert!(
9070                unresolved.error.as_ref().is_some_and(Error::is_retired_marker_deferred),
9071                "legacy/current/unproven markers must be deferred: {:?}",
9072                unresolved.error
9073            );
9074            assert!(unresolved.absence.is_none());
9075            for disk in before.iter().skip(12).flatten() {
9076                assert!(
9077                    disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9078                        .await
9079                        .is_ok()
9080                );
9081                disk.write_metadata("", bucket, object, original_marker.clone())
9082                    .await
9083                    .unwrap();
9084            }
9085        }
9086        set.disks.write().await[15] = None;
9087        let incomplete = store
9088            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9089            .await
9090            .unwrap();
9091        assert!(incomplete.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9092        assert!(incomplete.absence.is_none(), "offline targets cannot be omitted from a cleanup receipt");
9093        set.disks.write().await[15] = before[15].clone();
9094        let failure =
9095            crate::set_disk::DanglingDeleteFailure::install(bucket, object, 15, crate::disk::error::DiskError::FaultyDisk);
9096        let partial = store
9097            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9098            .await
9099            .unwrap();
9100        assert!(partial.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9101        assert!(
9102            partial.absence.is_none(),
9103            "fifteen successful/absent disks cannot hide one remaining marker"
9104        );
9105        assert!(
9106            before[15]
9107                .as_ref()
9108                .unwrap()
9109                .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9110                .await
9111                .is_ok()
9112        );
9113        drop(failure);
9114        let cleaned = store
9115            .heal_object_at_incarnation(bucket, object, &marker.to_string(), current, &heal)
9116            .await
9117            .expect("cleanup retired marker");
9118        assert!(cleaned.error.is_none(), "cleanup error: {:?}", cleaned.error);
9119        let proof = cleaned.absence.expect("complete cleanup receipt");
9120        assert!(proof.removed);
9121        assert_eq!(proof.bucket_incarnation_id, current);
9122        for disk in before.iter().flatten() {
9123            assert!(matches!(
9124                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9125                    .await,
9126                Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9127            ));
9128        }
9129        let replay = store
9130            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9131            .await
9132            .expect("idempotent replay");
9133        assert!(replay.error.is_none());
9134        assert!(!replay.absence.expect("authoritative absence receipt").removed);
9135        for (key, version, payload) in new_versions {
9136            let mut reader = store
9137                .handle_get_object_reader(
9138                    bucket,
9139                    key,
9140                    None,
9141                    HeaderMap::new(),
9142                    &ObjectOptions {
9143                        version_id: Some(version.to_string()),
9144                        ..Default::default()
9145                    },
9146                )
9147                .await
9148                .expect("new generation still readable");
9149            let mut actual = Vec::new();
9150            reader.stream.read_to_end(&mut actual).await.expect("read new bytes");
9151            assert_eq!(actual, payload);
9152        }
9153        assert_eq!(dirs.len(), 16);
9154    }
9155
9156    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9157    async fn current_marker_delete_ack_removes_stale_rejoined_copy() {
9158        let bucket = "current-marker-stale-rejoin";
9159        let object = "marker.bin";
9160        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
9161        let (_dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
9162        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
9163        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
9164        store
9165            .handle_make_bucket(
9166                bucket,
9167                &MakeBucketOptions {
9168                    versioning_enabled: true,
9169                    ..Default::default()
9170                },
9171            )
9172            .await
9173            .expect("create versioned bucket");
9174        let mut history = Vec::new();
9175        for value in 1..=3_u8 {
9176            let payload = vec![value; 4097];
9177            let version = store
9178                .put_object(
9179                    bucket,
9180                    object,
9181                    &mut PutObjReader::from_vec(payload.clone()),
9182                    &ObjectOptions {
9183                        versioned: true,
9184                        ..Default::default()
9185                    },
9186                )
9187                .await
9188                .expect("write the historical data versions")
9189                .version_id
9190                .expect("historical version must have an explicit identity");
9191            history.push((version, payload));
9192        }
9193        let marker = store
9194            .delete_object(
9195                bucket,
9196                object,
9197                ObjectOptions {
9198                    versioned: true,
9199                    ..Default::default()
9200                },
9201            )
9202            .await
9203            .expect("create current marker")
9204            .version_id
9205            .expect("marker has explicit version");
9206        let current = store
9207            .bucket_incarnation_id_from_disk(bucket)
9208            .await
9209            .expect("bucket incarnation");
9210        let set = &store.pools[0].disk_set[0];
9211        let before = set.disks.read().await.clone();
9212        for slot in 12..16 {
9213            set.disks.write().await[slot] = None;
9214        }
9215        store
9216            .delete_object(
9217                bucket,
9218                object,
9219                ObjectOptions {
9220                    versioned: true,
9221                    version_id: Some(marker.to_string()),
9222                    ..Default::default()
9223                },
9224            )
9225            .await
9226            .expect("acknowledged delete should remove the marker from the online quorum");
9227        *set.disks.write().await = before.clone();
9228
9229        for disk in before.iter().take(12).flatten() {
9230            assert!(matches!(
9231                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9232                    .await,
9233                Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9234            ));
9235        }
9236        for disk in before.iter().skip(12).flatten() {
9237            assert!(
9238                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9239                    .await
9240                    .is_ok(),
9241                "offline member must retain the stale current marker"
9242            );
9243        }
9244        let healed = store
9245            .heal_object_at_incarnation(
9246                bucket,
9247                object,
9248                &marker.to_string(),
9249                current,
9250                &rustfs_heal_contracts::heal_channel::HealOpts {
9251                    remove: true,
9252                    scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9253                    ..Default::default()
9254                },
9255            )
9256            .await
9257            .expect("stale current marker heal should complete");
9258        assert!(healed.error.is_none(), "stale current marker heal: {:?}", healed.error);
9259        assert!(healed.absence.as_ref().is_some_and(|proof| proof.removed));
9260        for disk in before.iter().flatten() {
9261            assert!(matches!(
9262                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9263                    .await,
9264                Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9265            ));
9266        }
9267        let replay = store
9268            .heal_object_at_incarnation(
9269                bucket,
9270                object,
9271                &marker.to_string(),
9272                current,
9273                &rustfs_heal_contracts::heal_channel::HealOpts {
9274                    remove: true,
9275                    scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9276                    ..Default::default()
9277                },
9278            )
9279            .await
9280            .expect("replay the exact removed marker identity");
9281        assert!(replay.error.is_none(), "exact marker replay: {:?}", replay.error);
9282        let absence = replay.absence.expect("exact replay must prove authoritative absence");
9283        assert!(!absence.removed, "replay must not claim another physical repair");
9284        assert_eq!(absence.bucket_incarnation_id, current);
9285        assert_eq!(absence.version_id, marker.to_string());
9286        let listed = store
9287            .clone()
9288            .inner_list_object_versions(bucket, "", None, None, None, 100)
9289            .await
9290            .expect("list surviving versions after marker cleanup");
9291        assert_eq!(listed.objects.len(), history.len(), "an absent marker is not enumerable in a new walk");
9292        assert!(listed.objects.iter().all(|info| !info.delete_marker));
9293        for (version, expected) in history {
9294            assert!(listed.objects.iter().any(|info| info.version_id == Some(version)));
9295            let mut reader = store
9296                .handle_get_object_reader(
9297                    bucket,
9298                    object,
9299                    None,
9300                    HeaderMap::new(),
9301                    &ObjectOptions {
9302                        version_id: Some(version.to_string()),
9303                        ..Default::default()
9304                    },
9305                )
9306                .await
9307                .expect("historical data must remain readable after cleanup and replay");
9308            let mut actual = Vec::new();
9309            reader
9310                .stream
9311                .read_to_end(&mut actual)
9312                .await
9313                .expect("read all historical bytes");
9314            assert_eq!(actual, expected);
9315        }
9316    }
9317
9318    #[tokio::test]
9319    async fn multipool_delete_marker_stays_with_existing_versions() {
9320        let bucket = "multipool-marker-routing";
9321        let object = "history.bin";
9322        let (_dirs, store) = multipool_version_test_store(bucket).await;
9323
9324        let mut expected_versions = Vec::new();
9325        for value in 1..=3_u8 {
9326            let written = store.pools[1]
9327                .put_object(
9328                    bucket,
9329                    object,
9330                    &mut PutObjReader::from_vec(vec![value; 4097]),
9331                    &ObjectOptions {
9332                        versioned: true,
9333                        user_defined: HashMap::from([
9334                            (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), format!("generation={value}")),
9335                            ("x-amz-meta-generation".to_string(), value.to_string()),
9336                        ]),
9337                        ..Default::default()
9338                    },
9339                )
9340                .await
9341                .expect("write a historical version deterministically to pool 1");
9342            expected_versions.push(written.version_id.expect("versioned PUT must acknowledge a UUID"));
9343        }
9344        let marker = store
9345            .delete_object(
9346                bucket,
9347                object,
9348                ObjectOptions {
9349                    versioned: true,
9350                    ..Default::default()
9351                },
9352            )
9353            .await
9354            .expect("delete the current version");
9355        assert!(marker.delete_marker);
9356        let marker_id = marker.version_id.expect("DELETE must acknowledge a marker UUID");
9357        assert!(!expected_versions.contains(&marker_id));
9358
9359        let local = store.pools[1]
9360            .clone()
9361            .inner_list_object_versions(bucket, object, None, None, None, 10)
9362            .await
9363            .expect("list the object-owning pool");
9364        assert_eq!(local.objects.len(), 4, "the marker must be committed beside the three existing versions");
9365        assert_eq!(local.objects[0].version_id, Some(marker_id));
9366        assert!(local.objects[0].delete_marker && local.objects[0].is_latest);
9367        let versions = store
9368            .clone()
9369            .inner_list_object_versions(bucket, object, None, None, None, 10)
9370            .await
9371            .expect("list all pools");
9372        assert_eq!(versions.objects.len(), 4);
9373        assert_eq!(versions.objects.iter().filter(|version| version.is_latest).count(), 1);
9374        for (index, version_id) in expected_versions.iter().copied().enumerate() {
9375            assert!(
9376                versions
9377                    .objects
9378                    .iter()
9379                    .any(|version| version.version_id == Some(version_id) && !version.is_latest)
9380            );
9381            let mut reader = store
9382                .handle_get_object_reader(
9383                    bucket,
9384                    object,
9385                    None,
9386                    HeaderMap::new(),
9387                    &ObjectOptions {
9388                        version_id: Some(version_id.to_string()),
9389                        ..Default::default()
9390                    },
9391                )
9392                .await
9393                .expect("historical version should remain readable");
9394            let mut payload = Vec::new();
9395            reader
9396                .stream
9397                .read_to_end(&mut payload)
9398                .await
9399                .expect("read all historical bytes");
9400            let value = u8::try_from(index + 1).expect("fixture generation fits u8");
9401            assert_eq!(payload, vec![value; 4097]);
9402            let listed = versions
9403                .objects
9404                .iter()
9405                .find(|version| version.version_id == Some(version_id))
9406                .expect("listed historical version");
9407            assert_eq!(listed.user_tags.as_str(), format!("generation={value}"));
9408            assert_eq!(listed.user_defined.get("x-amz-meta-generation"), Some(&value.to_string()));
9409        }
9410
9411        let repeated = store
9412            .delete_object(
9413                bucket,
9414                object,
9415                ObjectOptions {
9416                    versioned: true,
9417                    ..Default::default()
9418                },
9419            )
9420            .await
9421            .expect("repeat simple DELETE");
9422        assert!(repeated.delete_marker);
9423        assert_ne!(
9424            repeated.version_id,
9425            Some(marker_id),
9426            "each enabled-versioning DELETE creates a new marker"
9427        );
9428        store
9429            .delete_object(
9430                bucket,
9431                object,
9432                ObjectOptions {
9433                    versioned: true,
9434                    version_id: repeated.version_id.map(|id| id.to_string()),
9435                    ..Default::default()
9436                },
9437            )
9438            .await
9439            .expect("remove the newest marker by identity");
9440        store
9441            .delete_object(
9442                bucket,
9443                object,
9444                ObjectOptions {
9445                    versioned: true,
9446                    version_id: Some(marker_id.to_string()),
9447                    ..Default::default()
9448                },
9449            )
9450            .await
9451            .expect("remove the original marker by identity");
9452        let current = store
9453            .get_object_info(bucket, object, &ObjectOptions::default())
9454            .await
9455            .expect("previous version becomes current");
9456        assert_eq!(current.version_id, expected_versions.last().copied());
9457        assert!(!current.delete_marker);
9458    }
9459
9460    #[tokio::test]
9461    async fn multipool_existing_split_history_lists_and_paginates_without_metadata_writes() {
9462        for marker_pool in 0..2 {
9463            let bucket = format!("multipool-split-history-{marker_pool}");
9464            let object = "history.bin";
9465            let (dirs, store) = multipool_version_test_store(&bucket).await;
9466            let mut expected = Vec::new();
9467            for value in 1..=3_u8 {
9468                let version = store.pools[1 - marker_pool]
9469                    .put_object(
9470                        &bucket,
9471                        object,
9472                        &mut PutObjReader::from_vec(vec![value; 4097]),
9473                        &ObjectOptions {
9474                            versioned: true,
9475                            ..Default::default()
9476                        },
9477                    )
9478                    .await
9479                    .expect("seed history through the owning pool's normal write path");
9480                expected.push(version.version_id);
9481            }
9482            let marker = store.pools[marker_pool]
9483                .delete_object(
9484                    &bucket,
9485                    object,
9486                    ObjectOptions {
9487                        versioned: true,
9488                        ..Default::default()
9489                    },
9490                )
9491                .await
9492                .expect("reproduce the previously committed split marker with a normal pool DELETE");
9493            assert!(marker.delete_marker);
9494            expected.push(marker.version_id);
9495            expected.reverse();
9496            let mut before = Vec::new();
9497            for dir in &dirs {
9498                before.push(
9499                    tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9500                        .await
9501                        .expect("snapshot persisted version metadata"),
9502                );
9503            }
9504            for max_keys in [1, 2, 4, 10] {
9505                let mut key_marker = None;
9506                let mut version_marker = None;
9507                let mut listed = Vec::new();
9508                let mut completed = false;
9509                for _ in 0..6 {
9510                    let page = store
9511                        .clone()
9512                        .inner_list_object_versions(&bucket, object, key_marker.clone(), version_marker.clone(), None, max_keys)
9513                        .await
9514                        .expect("read a complete merged version page");
9515                    listed.extend(page.objects);
9516                    if !page.is_truncated {
9517                        completed = true;
9518                        break;
9519                    }
9520                    assert_ne!(
9521                        (&page.next_marker, &page.next_version_idmarker),
9522                        (&key_marker, &version_marker),
9523                        "version cursor must advance"
9524                    );
9525                    key_marker = page.next_marker;
9526                    version_marker = page.next_version_idmarker;
9527                }
9528                assert!(completed, "pagination must terminate");
9529                assert_eq!(listed.iter().map(|version| version.version_id).collect::<Vec<_>>(), expected);
9530                assert!(listed[0].delete_marker && listed[0].is_latest);
9531                assert!(listed[1..].iter().all(|version| !version.delete_marker && !version.is_latest));
9532            }
9533            let visible = store
9534                .clone()
9535                .list_objects_generic(&bucket, "", None, None, 10, false)
9536                .await
9537                .expect("list current objects");
9538            assert!(visible.objects.is_empty(), "the global current marker hides the object");
9539            for (dir, before) in dirs.iter().zip(before) {
9540                assert_eq!(
9541                    tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9542                        .await
9543                        .expect("read unchanged metadata"),
9544                    before
9545                );
9546            }
9547        }
9548    }
9549
9550    #[tokio::test]
9551    async fn multipool_conflicting_version_metadata_fails_the_listing_request() {
9552        let bucket = "multipool-version-conflict";
9553        let (_dirs, store) = multipool_version_test_store(bucket).await;
9554        store.pools[0]
9555            .put_object(
9556                bucket,
9557                "a.bin",
9558                &mut PutObjReader::from_vec(b"preceding result".to_vec()),
9559                &ObjectOptions {
9560                    versioned: true,
9561                    ..Default::default()
9562                },
9563            )
9564            .await
9565            .expect("seed an entry before the conflict");
9566        let version_id = Uuid::new_v4();
9567        let mod_time = OffsetDateTime::now_utc();
9568        let mut etags = Vec::new();
9569        for (pool_idx, value) in [(0, 1), (1, 2)] {
9570            let written = store.pools[pool_idx]
9571                .put_object(
9572                    bucket,
9573                    "z.bin",
9574                    &mut PutObjReader::from_vec(vec![value; 4097]),
9575                    &ObjectOptions {
9576                        versioned: true,
9577                        version_id: Some(version_id.to_string()),
9578                        mod_time: Some(mod_time),
9579                        ..Default::default()
9580                    },
9581                )
9582                .await
9583                .expect("persist independent conflicting copies");
9584            assert_eq!(written.version_id, Some(version_id));
9585            etags.push(written.etag);
9586        }
9587        assert_ne!(etags[0], etags[1], "fixture must contain a semantic conflict");
9588        let err = store
9589            .clone()
9590            .inner_list_object_versions(bucket, "", None, None, None, 10)
9591            .await
9592            .expect_err("partial output must not hide the merge error");
9593        assert_eq!(err, StorageError::FileCorrupt);
9594
9595        let cancellation = tokio_util::sync::CancellationToken::new();
9596        let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
9597        let walk = store
9598            .clone()
9599            .walk(cancellation.clone(), bucket, "", sender, WalkOptions::default());
9600        let drain = async { while receiver.recv().await.is_some() {} };
9601        let (walk_result, ()) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(walk, drain) })
9602            .await
9603            .expect("bounded walk output must drain and terminate on a merge error");
9604        assert_eq!(
9605            walk_result.expect_err("walk must report the same metadata conflict"),
9606            StorageError::FileCorrupt
9607        );
9608        assert!(!cancellation.is_cancelled(), "worker failure must not cancel the caller's request");
9609    }
9610
9611    #[tokio::test]
9612    async fn multipool_marker_rejects_unwritable_owner_without_falling_back() {
9613        let bucket = "multipool-unwritable-owner";
9614        let object = "history.bin";
9615        let (_dirs, store) = multipool_version_test_store(bucket).await;
9616        store.pools[1]
9617            .put_object(
9618                bucket,
9619                object,
9620                &mut PutObjReader::from_vec(vec![1; 4097]),
9621                &ObjectOptions {
9622                    versioned: true,
9623                    ..Default::default()
9624                },
9625            )
9626            .await
9627            .expect("seed the nonzero owner");
9628        *store.pool_meta.write().await = PoolMeta {
9629            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
9630            ..Default::default()
9631        };
9632        let err = store
9633            .delete_object(
9634                bucket,
9635                object,
9636                ObjectOptions {
9637                    versioned: true,
9638                    ..Default::default()
9639                },
9640            )
9641            .await
9642            .expect_err("suspended owner must reject marker creation");
9643        assert_eq!(err, StorageError::SlowDown);
9644        *store.pool_meta.write().await = PoolMeta::default();
9645
9646        let mut rebalancing = crate::services::rebalance::RebalanceStats {
9647            participating: true,
9648            ..Default::default()
9649        };
9650        rebalancing.info.status = crate::services::rebalance::RebalStatus::Started;
9651        *store.rebalance_meta.write().await = Some(crate::services::rebalance::RebalanceMeta {
9652            pool_stats: vec![crate::services::rebalance::RebalanceStats::default(), rebalancing],
9653            ..Default::default()
9654        });
9655        let error = store
9656            .delete_object(
9657                bucket,
9658                object,
9659                ObjectOptions {
9660                    versioned: true,
9661                    ..Default::default()
9662                },
9663            )
9664            .await;
9665        *store.rebalance_meta.write().await = None;
9666        assert_eq!(
9667            error.expect_err("rebalance must not hide the owner during marker lookup"),
9668            StorageError::SlowDown
9669        );
9670
9671        // Isolate object quorum from the bucket metadata preflight: disabling
9672        // a whole pool can otherwise fail before object ownership is looked up.
9673        let quorum_bucket = RUSTFS_META_BUCKET;
9674        store.pools[1]
9675            .put_object(
9676                quorum_bucket,
9677                object,
9678                &mut PutObjReader::from_vec(vec![1; 4097]),
9679                &ObjectOptions {
9680                    versioned: true,
9681                    ..Default::default()
9682                },
9683            )
9684            .await
9685            .expect("seed the object-quorum fixture");
9686        for pool in &store.pools {
9687            pool.put_object(
9688                quorum_bucket,
9689                "split.bin",
9690                &mut PutObjReader::from_vec(vec![2; 4097]),
9691                &ObjectOptions {
9692                    versioned: true,
9693                    ..Default::default()
9694                },
9695            )
9696            .await
9697            .expect("seed another key with history in both pools");
9698        }
9699
9700        let owner = &store.pools[1].disk_set[0];
9701        let healthy = owner.disks.read().await.clone();
9702        for disk in owner.disks.write().await.iter_mut().skip(1) {
9703            *disk = None;
9704        }
9705        let error = store
9706            .delete_object(
9707                quorum_bucket,
9708                object,
9709                ObjectOptions {
9710                    versioned: true,
9711                    ..Default::default()
9712                },
9713            )
9714            .await;
9715        let split_error = store
9716            .delete_object(
9717                quorum_bucket,
9718                "split.bin",
9719                ObjectOptions {
9720                    versioned: true,
9721                    ..Default::default()
9722                },
9723            )
9724            .await;
9725        *owner.disks.write().await = healthy;
9726        assert_eq!(
9727            error.expect_err("subquorum owner must not become a new-pool marker"),
9728            StorageError::ErasureWriteQuorum
9729        );
9730        assert_eq!(
9731            split_error.expect_err("a readable older pool does not prove the global current version"),
9732            StorageError::ErasureWriteQuorum
9733        );
9734        let split = store.pools[0]
9735            .clone()
9736            .inner_list_object_versions(quorum_bucket, "split.bin", None, None, None, 10)
9737            .await
9738            .expect("inspect the readable older pool");
9739        assert_eq!(split.objects.len(), 1);
9740        assert!(!split.objects[0].delete_marker);
9741        let other = store.pools[0]
9742            .clone()
9743            .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9744            .await
9745            .expect("inspect the other pool");
9746        assert!(other.objects.is_empty());
9747        let history = store.pools[1]
9748            .clone()
9749            .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9750            .await
9751            .expect("inspect the restored owner");
9752        assert_eq!(history.objects.len(), 1);
9753        assert!(!history.objects[0].delete_marker);
9754    }
9755
9756    #[tokio::test]
9757    async fn multipool_suspended_and_batch_markers_keep_the_null_slot_on_the_owner() {
9758        let bucket = "multipool-suspended-markers";
9759        let object = "history.bin";
9760        let (_dirs, store) = multipool_version_test_store(bucket).await;
9761        let original = store.pools[1]
9762            .put_object(
9763                bucket,
9764                object,
9765                &mut PutObjReader::from_vec(vec![1; 4097]),
9766                &ObjectOptions {
9767                    versioned: true,
9768                    ..Default::default()
9769                },
9770            )
9771            .await
9772            .expect("seed the UUID history in pool 1");
9773        store
9774            .update_bucket_metadata_config(
9775                bucket,
9776                crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
9777                b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
9778            )
9779            .await
9780            .expect("persist suspended bucket versioning");
9781        let null = store
9782            .put_object(
9783                bucket,
9784                object,
9785                &mut PutObjReader::from_vec(vec![2; 4097]),
9786                &ObjectOptions {
9787                    version_suspended: true,
9788                    ..Default::default()
9789                },
9790            )
9791            .await
9792            .expect("write the null version beside the history");
9793        assert!(null.version_id.is_none_or(|id| id.is_nil()));
9794        for _ in 0..2 {
9795            let marker = store
9796                .delete_object(
9797                    bucket,
9798                    object,
9799                    ObjectOptions {
9800                        version_suspended: true,
9801                        ..Default::default()
9802                    },
9803                )
9804                .await
9805                .expect("replace the null slot with a marker");
9806            assert!(marker.delete_marker);
9807            assert!(marker.version_id.is_none_or(|id| id.is_nil()));
9808        }
9809        let (deleted, errors) = store
9810            .delete_objects(
9811                bucket,
9812                vec![ObjectToDelete {
9813                    object_name: object.to_string(),
9814                    ..Default::default()
9815                }],
9816                ObjectOptions::default(),
9817            )
9818            .await;
9819        assert!(errors.iter().all(Option::is_none), "batch DELETE should succeed: {errors:?}");
9820        assert_eq!(deleted.len(), 1);
9821        assert!(deleted[0].delete_marker);
9822        let versions = store.pools[1]
9823            .clone()
9824            .inner_list_object_versions(bucket, object, None, None, None, 10)
9825            .await
9826            .expect("inspect the owner after batch DELETE");
9827        assert_eq!(versions.objects.len(), 2, "only one null marker and the UUID history remain");
9828        assert!(versions.objects[0].delete_marker && versions.objects[0].is_latest);
9829        assert_eq!(versions.objects[1].version_id, original.version_id);
9830        let other = store.pools[0]
9831            .clone()
9832            .inner_list_object_versions(bucket, object, None, None, None, 10)
9833            .await
9834            .expect("inspect the unused pool");
9835        assert!(other.objects.is_empty());
9836    }
9837
9838    async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
9839        assert_pool_writer_is_blocked(store, 0, bucket, object).await;
9840    }
9841
9842    async fn assert_pool_writer_is_blocked(store: &ECStore, pool_idx: usize, bucket: &str, object: &str) {
9843        let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9844        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9845        let err = lock
9846            .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
9847            .await
9848            .expect_err("prepared read lock should block the writer");
9849        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
9850    }
9851
9852    async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
9853        acquire_pool_writer(store, 0, bucket, object).await
9854    }
9855
9856    async fn acquire_pool_writer(
9857        store: &ECStore,
9858        pool_idx: usize,
9859        bucket: &str,
9860        object: &str,
9861    ) -> rustfs_lock::NamespaceLockGuard {
9862        let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9863        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9864        lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
9865            .await
9866            .expect("prepared read lock should have been released")
9867    }
9868
9869    #[tokio::test]
9870    #[serial_test::serial(body_cache_hook)]
9871    async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
9872        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9873        let store = new_prepared_reader_test_store(&[set_disks]).await;
9874        let bucket = "prepared-reader-hook-miss";
9875        let object = "object.bin";
9876        let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
9877        let opts = ObjectOptions {
9878            no_lock: true,
9879            ..Default::default()
9880        };
9881
9882        store.pools[0]
9883            .make_bucket(bucket, &MakeBucketOptions::default())
9884            .await
9885            .expect("bucket should be created");
9886        let mut put_reader = PutObjReader::from_vec(payload.clone());
9887        store.pools[0]
9888            .put_object(bucket, object, &mut put_reader, &opts)
9889            .await
9890            .expect("object should be written");
9891
9892        clear_get_object_body_cache_hook();
9893        let hook = Arc::new(CountingMissHook {
9894            calls: AtomicUsize::new(0),
9895        });
9896        register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
9897        let _hook_guard = BodyCacheHookGuard;
9898
9899        let prepared = store
9900            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
9901            .await
9902            .expect("prepared reader metadata should resolve");
9903        assert!(matches!(
9904            lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
9905            GetObjectBodyCacheHookLookup::Miss
9906        ));
9907        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");
9908
9909        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9910        assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
9911        let mut restored = Vec::new();
9912        reader
9913            .stream
9914            .read_to_end(&mut restored)
9915            .await
9916            .expect("prepared body should stream");
9917
9918        assert_eq!(restored, payload);
9919        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
9920    }
9921
9922    #[tokio::test]
9923    #[serial_test::serial(body_cache_hook)]
9924    async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
9925        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
9926            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9927            let store = new_prepared_reader_test_store(&[set_disks]).await;
9928            let bucket = "prepared-reader-lock-lifetime";
9929            let object = "object.bin";
9930            let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
9931            let put_opts = ObjectOptions {
9932                no_lock: true,
9933                ..Default::default()
9934            };
9935
9936            store.pools[0]
9937                .make_bucket(bucket, &MakeBucketOptions::default())
9938                .await
9939                .expect("bucket should be created");
9940            let mut put_reader = PutObjReader::from_vec(payload.clone());
9941            store.pools[0]
9942                .put_object(bucket, object, &mut put_reader, &put_opts)
9943                .await
9944                .expect("object should be written");
9945
9946            let prepared = store
9947                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9948                .await
9949                .expect("prepared reader metadata should resolve");
9950            assert!(prepared.read_lock_guard.is_some());
9951            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9952
9953            let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9954            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9955            let mut restored = Vec::new();
9956            reader
9957                .stream
9958                .read_to_end(&mut restored)
9959                .await
9960                .expect("prepared body should stream");
9961            assert_eq!(restored, payload);
9962            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9963
9964            let prepared = store
9965                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9966                .await
9967                .expect("second prepared reader metadata should resolve");
9968            let reader = prepared.into_reader().await.expect("second prepared body reader should open");
9969            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9970            drop(reader);
9971            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9972        })
9973        .await;
9974    }
9975
9976    #[tokio::test]
9977    #[serial_test::serial(body_cache_hook)]
9978    async fn select_snapshot_holds_namespace_lock_independent_of_get_optimization() {
9979        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
9980            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9981            let store = new_prepared_reader_test_store(&[set_disks]).await;
9982            let bucket = "select-snapshot-lock-lifetime";
9983            let object = "object.bin";
9984            let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
9985            let put_opts = ObjectOptions {
9986                no_lock: true,
9987                ..Default::default()
9988            };
9989
9990            store.pools[0]
9991                .make_bucket(bucket, &MakeBucketOptions::default())
9992                .await
9993                .expect("bucket should be created");
9994            let mut put_reader = PutObjReader::from_vec(payload.clone());
9995            store.pools[0]
9996                .put_object(bucket, object, &mut put_reader, &put_opts)
9997                .await
9998                .expect("object should be written");
9999
10000            use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
10001            let mut request_headers = HeaderMap::new();
10002            request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
10003            request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
10004            request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
10005            request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
10006            let snapshot = store
10007                .prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
10008                .await
10009                .expect("SelectObjectContent snapshot should be prepared");
10010            assert_eq!(snapshot.headers.len(), 3);
10011            assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
10012            assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
10013            assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
10014            assert!(snapshot.headers.get("authorization").is_none());
10015            assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
10016            assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
10017            assert_eq!(
10018                snapshot.logical_size(),
10019                u64::try_from(payload.len()).expect("test payload length should fit in u64")
10020            );
10021            assert_eq!(
10022                snapshot.object_info().size,
10023                i64::try_from(payload.len()).expect("test payload length should fit in i64")
10024            );
10025            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10026
10027            let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10028            let mut restored = Vec::new();
10029            reader
10030                .stream
10031                .read_to_end(&mut restored)
10032                .await
10033                .expect("snapshot body should stream");
10034            assert_eq!(restored, payload);
10035            drop(reader);
10036            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10037
10038            drop(snapshot);
10039            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10040        })
10041        .await;
10042    }
10043
10044    #[tokio::test]
10045    async fn select_snapshot_identity_compares_encoded_directory_object_key() {
10046        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10047        let store = new_prepared_reader_test_store(&[set_disks]).await;
10048        let bucket = "select-snapshot-directory-identity";
10049        let source_object = "source.bin";
10050
10051        store.pools[0]
10052            .make_bucket(bucket, &MakeBucketOptions::default())
10053            .await
10054            .expect("bucket should be created");
10055        let mut put_reader = PutObjReader::from_vec(b"source".to_vec());
10056        store.pools[0]
10057            .put_object(
10058                bucket,
10059                source_object,
10060                &mut put_reader,
10061                &ObjectOptions {
10062                    no_lock: true,
10063                    ..Default::default()
10064                },
10065            )
10066            .await
10067            .expect("source object should be written");
10068
10069        let mut snapshot = store
10070            .prepare_select_object_snapshot(bucket, source_object, &HeaderMap::new(), &ObjectOptions::default())
10071            .await
10072            .expect("source object snapshot should be prepared");
10073        snapshot.object = encode_dir_object("directory/");
10074
10075        assert!(snapshot.is_for(bucket, "directory/"));
10076        assert!(!snapshot.is_for(bucket, "different/"));
10077    }
10078
10079    #[tokio::test]
10080    #[serial_test::serial(body_cache_hook)]
10081    async fn select_snapshot_reuses_initial_metadata_fanout_for_first_reader() {
10082        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10083        let store = new_prepared_reader_test_store(&[set_disks]).await;
10084        let bucket = "select-snapshot-initial-metadata";
10085        let object = "initial-metadata.bin";
10086        let payload = b"select snapshot initial metadata".repeat(4_000);
10087        let no_lock_opts = ObjectOptions {
10088            no_lock: true,
10089            ..Default::default()
10090        };
10091
10092        store.pools[0]
10093            .make_bucket(bucket, &MakeBucketOptions::default())
10094            .await
10095            .expect("bucket should be created");
10096        let mut put_reader = PutObjReader::from_vec(payload.clone());
10097        store.pools[0]
10098            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10099            .await
10100            .expect("object should be written");
10101
10102        let calls = disk_call_counters::observe(object);
10103        let snapshot = store
10104            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10105            .await
10106            .expect("snapshot should prepare metadata");
10107        assert_eq!(
10108            calls.total(disk_call_counters::KIND_READ_VERSION),
10109            4,
10110            "snapshot preparation should fan out metadata once"
10111        );
10112
10113        let mut reader = snapshot.open_reader(None).await.expect("first snapshot reader should open");
10114        assert_eq!(
10115            calls.total(disk_call_counters::KIND_READ_VERSION),
10116            4,
10117            "first reader should consume the metadata captured during preparation"
10118        );
10119        let mut restored = Vec::new();
10120        reader
10121            .stream
10122            .read_to_end(&mut restored)
10123            .await
10124            .expect("snapshot body should stream");
10125        assert_eq!(restored, payload);
10126    }
10127
10128    #[tokio::test]
10129    #[serial_test::serial(body_cache_hook)]
10130    async fn select_snapshot_rejects_identity_change_before_second_reader() {
10131        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10132        let store = new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await;
10133        let bucket = "select-snapshot-identity-change";
10134        let object = "identity-change.bin";
10135        let no_lock_opts = ObjectOptions {
10136            no_lock: true,
10137            ..Default::default()
10138        };
10139
10140        set_disks
10141            .make_bucket(bucket, &MakeBucketOptions::default())
10142            .await
10143            .expect("bucket should be created");
10144        let mut initial_reader = PutObjReader::from_vec(b"first generation".to_vec());
10145        set_disks
10146            .put_object(bucket, object, &mut initial_reader, &no_lock_opts)
10147            .await
10148            .expect("initial generation should be written");
10149        let mut snapshot = store
10150            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10151            .await
10152            .expect("snapshot should be prepared");
10153        drop(
10154            snapshot
10155                .open_reader(None)
10156                .await
10157                .expect("first snapshot reader should consume captured metadata"),
10158        );
10159        snapshot.opts.metadata_cache_safe = false;
10160
10161        let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10162        set_disks
10163            .put_object(bucket, object, &mut replacement_reader, &no_lock_opts)
10164            .await
10165            .expect("test-only no-lock write should replace the object");
10166        let error = match snapshot.open_reader(None).await {
10167            Ok(_) => panic!("a later reader must reject the replacement generation"),
10168            Err(error) => error,
10169        };
10170        assert!(matches!(
10171            error,
10172            SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged)
10173        ));
10174    }
10175
10176    #[tokio::test]
10177    #[serial_test::serial(body_cache_hook)]
10178    async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() {
10179        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10180        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10181        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10182        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10183        let bucket = "select-snapshot-latest-delete-marker";
10184        let object = "versioned-object.bin";
10185        let versioned_opts = ObjectOptions {
10186            no_lock: true,
10187            versioned: true,
10188            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10189            ..Default::default()
10190        };
10191
10192        for set_disks in [&first_set, &second_set] {
10193            set_disks
10194                .make_bucket(bucket, &MakeBucketOptions::default())
10195                .await
10196                .expect("bucket should be created");
10197        }
10198        let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10199        first_set
10200            .put_object(bucket, object, &mut older_reader, &versioned_opts)
10201            .await
10202            .expect("older versioned object should be written");
10203        let mut put_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10204        second_set
10205            .put_object(bucket, object, &mut put_reader, &versioned_opts)
10206            .await
10207            .expect("versioned object should be written");
10208        let marker = second_set
10209            .delete_object(bucket, object, versioned_opts.clone())
10210            .await
10211            .expect("versioned delete should create a marker");
10212        assert!(marker.delete_marker);
10213        assert!(marker.version_id.is_some_and(|version_id| !version_id.is_nil()));
10214
10215        let error = store
10216            .prepare_select_object_snapshot(
10217                bucket,
10218                object,
10219                &HeaderMap::new(),
10220                &ObjectOptions {
10221                    versioned: true,
10222                    ..Default::default()
10223                },
10224            )
10225            .await
10226            .expect_err("latest delete marker should hide the prior object generation");
10227        assert!(matches!(
10228            error,
10229            PrepareSelectObjectSnapshotError::Storage(ref error) if is_err_object_not_found(error)
10230        ));
10231    }
10232
10233    #[tokio::test]
10234    #[serial_test::serial(storage_class_env)]
10235    async fn select_snapshot_blocks_store_delete_for_object_in_nonzero_set() {
10236        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10237        let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
10238        ctx.update_erasure_type(SetupType::DistErasure).await;
10239        assert!(
10240            sets.disk_set[0]
10241                .lockers
10242                .iter()
10243                .all(|first| sets.disk_set[1].lockers.iter().all(|second| !Arc::ptr_eq(first, second)))
10244        );
10245        let pool_config = sets.endpoints.clone();
10246        let store = Arc::new(new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx));
10247        let bucket = RUSTFS_META_BUCKET;
10248        let object = (0..1_000)
10249            .map(|index| format!("nonzero-set-{index}.bin"))
10250            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
10251            .expect("a key should hash to the second set");
10252        let mut put_reader = PutObjReader::from_vec(b"stable snapshot body".to_vec());
10253        sets.put_object(
10254            bucket,
10255            &object,
10256            &mut put_reader,
10257            &ObjectOptions {
10258                no_lock: true,
10259                ..Default::default()
10260            },
10261        )
10262        .await
10263        .expect("object should be written to the second set");
10264        assert!(Arc::ptr_eq(&sets.get_disks_by_key(&object), &sets.disk_set[1]));
10265
10266        let snapshot = store
10267            .prepare_select_object_snapshot(bucket, &object, &HeaderMap::new(), &ObjectOptions::default())
10268            .await
10269            .expect("snapshot should acquire both lock domains");
10270        let hashed_set_writer = rustfs_lock::NamespaceLock::with_clients_and_quorum(
10271            "select-nonzero-set-writer".to_string(),
10272            sets.disk_set[1].lockers.clone(),
10273            2,
10274        );
10275        let writer_error = hashed_set_writer
10276            .get_write_lock(
10277                rustfs_lock::ObjectKey::new(bucket, object.as_str()),
10278                "competing-hashed-set-writer",
10279                Duration::from_millis(50),
10280            )
10281            .await
10282            .expect_err("Select must hold the nonzero hashed-set read lock");
10283        assert!(matches!(writer_error, rustfs_lock::LockError::Timeout { .. }));
10284        let barrier = DeleteAfterObjectLockSnapshotBarrier::install(bucket);
10285        let delete_store = Arc::clone(&store);
10286        let delete_object = object.clone();
10287        let mut delete = tokio::spawn(async move {
10288            delete_store
10289                .delete_object(bucket, &delete_object, ObjectOptions::default())
10290                .await
10291        });
10292        barrier.wait_until_paused().await;
10293        barrier.release();
10294        assert!(
10295            tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(),
10296            "store DELETE must wait for the fixed-domain Select read lock"
10297        );
10298
10299        drop(snapshot);
10300        let deleted = tokio::time::timeout(Duration::from_secs(10), delete)
10301            .await
10302            .expect("DELETE should resume after the snapshot is dropped")
10303            .expect("DELETE task should not panic")
10304            .expect("DELETE should complete");
10305        assert_eq!(deleted.name, object);
10306    }
10307
10308    #[tokio::test]
10309    #[serial_test::serial]
10310    async fn select_snapshot_fails_closed_when_local_locking_is_disabled() {
10311        let ctx = Arc::new(crate::runtime::instance::InstanceContext::with_lock_manager_for_test(Arc::new(
10312            rustfs_lock::GlobalLockManager::Disabled(rustfs_lock::DisabledLockManager::new()),
10313        )));
10314        let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10315        let store = new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await;
10316        let bucket = "select-snapshot-lock-disabled";
10317        let object = "object.bin";
10318        let mut put_reader = PutObjReader::from_vec(b"payload".to_vec());
10319        let no_lock_opts = ObjectOptions {
10320            no_lock: true,
10321            ..Default::default()
10322        };
10323
10324        store.pools[0]
10325            .make_bucket(bucket, &MakeBucketOptions::default())
10326            .await
10327            .expect("bucket should be created");
10328        store.pools[0]
10329            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10330            .await
10331            .expect("object should be written without namespace locking");
10332
10333        let error = store
10334            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10335            .await
10336            .expect_err("SelectObjectContent must reject a disabled lock manager");
10337        assert!(matches!(
10338            error,
10339            PrepareSelectObjectSnapshotError::Consistency(SnapshotConsistencyError::LockingDisabled)
10340        ));
10341    }
10342
10343    #[tokio::test]
10344    #[serial_test::serial(body_cache_hook)]
10345    async fn prepared_object_info_releases_namespace_lock_immediately() {
10346        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10347        let store = new_prepared_reader_test_store(&[set_disks]).await;
10348        let bucket = "prepared-object-info-lock-release";
10349        let object = "object.bin";
10350        let payload = b"prepared-object-info-lock-release".to_vec();
10351        let put_opts = ObjectOptions {
10352            no_lock: true,
10353            ..Default::default()
10354        };
10355
10356        store.pools[0]
10357            .make_bucket(bucket, &MakeBucketOptions::default())
10358            .await
10359            .expect("bucket should be created");
10360        let mut put_reader = PutObjReader::from_vec(payload.clone());
10361        store.pools[0]
10362            .put_object(bucket, object, &mut put_reader, &put_opts)
10363            .await
10364            .expect("object should be written");
10365
10366        let prepared = store
10367            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
10368            .await
10369            .expect("prepared reader metadata should resolve");
10370        assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10371        assert_eq!(prepared.into_object_info().size, payload.len() as i64);
10372
10373        drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10374    }
10375
10376    #[tokio::test]
10377    #[serial_test::serial(body_cache_hook)]
10378    async fn prepared_reader_reuses_metadata_across_three_pools() {
10379        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10380        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10381        let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10382        let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10383        let bucket = "prepared-reader-three-pools";
10384        let object = "object.bin";
10385        let payload = b"prepared-reader-three-pool-payload-".repeat(40_000);
10386        let opts = ObjectOptions {
10387            no_lock: true,
10388            ..Default::default()
10389        };
10390
10391        for pool in &store.pools {
10392            pool.make_bucket(bucket, &MakeBucketOptions::default())
10393                .await
10394                .expect("bucket should be created in each pool");
10395        }
10396        let mut put_reader = PutObjReader::from_vec(payload.clone());
10397        store.pools[2]
10398            .put_object(bucket, object, &mut put_reader, &opts)
10399            .await
10400            .expect("object should be written only to the third pool");
10401
10402        let calls = disk_call_counters::observe(object);
10403        let prepared = store
10404            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10405            .await
10406            .expect("prepared reader should resolve the third-pool object");
10407        assert_eq!(prepared.object_info().size, payload.len() as i64);
10408        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10409        assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10410        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
10411        assert_eq!(
10412            calls.total(disk_call_counters::KIND_READ_VERSION),
10413            metadata_calls,
10414            "the selected pool must reuse its prepared metadata"
10415        );
10416        let mut restored = Vec::new();
10417        reader
10418            .stream
10419            .read_to_end(&mut restored)
10420            .await
10421            .expect("prepared body should stream");
10422        assert_eq!(restored, payload);
10423    }
10424
10425    #[tokio::test]
10426    #[serial_test::serial(body_cache_hook)]
10427    async fn select_snapshot_reuses_metadata_across_three_pools() {
10428        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10429        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10430        let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10431        let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10432        let bucket = "select-snapshot-three-pools";
10433        let object = "object.bin";
10434        let payload = b"select-snapshot-three-pool-payload-".repeat(40_000);
10435        let write_opts = ObjectOptions {
10436            no_lock: true,
10437            ..Default::default()
10438        };
10439
10440        for pool in &store.pools {
10441            pool.make_bucket(bucket, &MakeBucketOptions::default())
10442                .await
10443                .expect("bucket should be created in each pool");
10444        }
10445        let mut put_reader = PutObjReader::from_vec(payload.clone());
10446        store.pools[2]
10447            .put_object(bucket, object, &mut put_reader, &write_opts)
10448            .await
10449            .expect("object should be written only to the third pool");
10450
10451        let calls = disk_call_counters::observe(object);
10452        let snapshot = store
10453            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10454            .await
10455            .expect("SelectObjectContent snapshot should resolve the third-pool object");
10456        assert_eq!(snapshot.object_info().size, payload.len() as i64);
10457        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10458        assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10459
10460        let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open");
10461        assert_eq!(
10462            calls.total(disk_call_counters::KIND_READ_VERSION),
10463            metadata_calls,
10464            "SelectObjectContent must consume the prepared winner without a second fanout"
10465        );
10466        let mut restored = Vec::new();
10467        reader
10468            .stream
10469            .read_to_end(&mut restored)
10470            .await
10471            .expect("snapshot body should stream");
10472        assert_eq!(restored, payload);
10473    }
10474
10475    #[tokio::test]
10476    #[serial_test::serial(body_cache_hook)]
10477    async fn legacy_reader_reuses_selected_pool_metadata() {
10478        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10479        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10480        let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10481        let bucket = "legacy-reader-second-pool";
10482        let object = "object.bin";
10483        let payload = b"legacy-reader-second-pool-payload-".repeat(40_000);
10484        let opts = ObjectOptions {
10485            no_lock: true,
10486            ..Default::default()
10487        };
10488
10489        for pool in &store.pools {
10490            pool.make_bucket(bucket, &MakeBucketOptions::default())
10491                .await
10492                .expect("bucket should be created in each pool");
10493        }
10494        let mut put_reader = PutObjReader::from_vec(payload.clone());
10495        store.pools[1]
10496            .put_object(bucket, object, &mut put_reader, &opts)
10497            .await
10498            .expect("object should be written only to the second pool");
10499
10500        clear_get_object_body_cache_hook();
10501        let hook = Arc::new(CountingMissHook {
10502            calls: AtomicUsize::new(0),
10503        });
10504        register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
10505        let _hook_guard = BodyCacheHookGuard;
10506
10507        let calls = disk_call_counters::observe(object);
10508        let mut reader = store
10509            .handle_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10510            .await
10511            .expect("legacy reader should resolve the second-pool object");
10512        assert_eq!(
10513            hook.calls.load(Ordering::Relaxed),
10514            1,
10515            "legacy reader must probe the body cache exactly once"
10516        );
10517        assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
10518        assert!(
10519            calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10520            "legacy reader must fan out each 4-disk pool at most once"
10521        );
10522        let mut restored = Vec::new();
10523        reader
10524            .stream
10525            .read_to_end(&mut restored)
10526            .await
10527            .expect("legacy reader body should stream");
10528        assert_eq!(restored, payload);
10529    }
10530
10531    fn prepared_pool_test_status(id: usize, suspended: bool) -> PoolStatus {
10532        PoolStatus {
10533            id,
10534            cmd_line: format!("prepared-pool-{id}"),
10535            last_update: OffsetDateTime::now_utc(),
10536            decommission: suspended.then(|| PoolDecommissionInfo {
10537                start_time: Some(OffsetDateTime::now_utc()),
10538                ..Default::default()
10539            }),
10540        }
10541    }
10542
10543    #[tokio::test]
10544    #[serial_test::serial(body_cache_hook)]
10545    async fn prepared_reader_refetches_when_final_pool_state_changes_winner() {
10546        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10547        let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10548        let bucket = "prepared-reader-pool-state-fallback";
10549        let object = "object.bin";
10550        let payload = b"pool-state fallback payload".repeat(8_000);
10551        let opts = ObjectOptions {
10552            no_lock: true,
10553            ..Default::default()
10554        };
10555
10556        set_disks
10557            .make_bucket(bucket, &MakeBucketOptions::default())
10558            .await
10559            .expect("bucket should be created");
10560        let mut put_reader = PutObjReader::from_vec(payload.clone());
10561        set_disks
10562            .put_object(bucket, object, &mut put_reader, &opts)
10563            .await
10564            .expect("shared object should be written");
10565
10566        let calls = disk_call_counters::observe(object);
10567        let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, false);
10568        let read_store = Arc::clone(&store);
10569        let read_opts = opts.clone();
10570        let read = tokio::spawn(async move {
10571            read_store
10572                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10573                .await
10574        });
10575        barrier.wait_after_fanout().await;
10576        *store.pool_meta.write().await = PoolMeta {
10577            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10578            ..Default::default()
10579        };
10580        barrier.release_after_fanout();
10581
10582        let prepared = read
10583            .await
10584            .expect("prepared read task should not panic")
10585            .expect("final active pool should be refetched");
10586        assert!(Arc::ptr_eq(&prepared.pool, &store.pools[0]));
10587        assert_eq!(
10588            calls.total(disk_call_counters::KIND_READ_VERSION),
10589            12,
10590            "two initial 4-disk fanouts plus one fallback refetch are required"
10591        );
10592        let mut reader = prepared.into_reader().await.expect("fallback body reader should open");
10593        let mut restored = Vec::new();
10594        reader
10595            .stream
10596            .read_to_end(&mut restored)
10597            .await
10598            .expect("fallback body should stream");
10599        assert_eq!(restored, payload);
10600    }
10601
10602    #[tokio::test]
10603    #[serial_test::serial(body_cache_hook)]
10604    async fn prepared_reader_fallback_rejects_generation_change_before_refetch() {
10605        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10606        let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10607        let bucket = "prepared-reader-pool-state-generation-change";
10608        let object = "object.bin";
10609        let opts = ObjectOptions {
10610            no_lock: true,
10611            ..Default::default()
10612        };
10613
10614        set_disks
10615            .make_bucket(bucket, &MakeBucketOptions::default())
10616            .await
10617            .expect("bucket should be created");
10618        let mut initial_reader = PutObjReader::from_vec(b"initial generation".to_vec());
10619        set_disks
10620            .put_object(bucket, object, &mut initial_reader, &opts)
10621            .await
10622            .expect("initial object should be written");
10623
10624        let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, true);
10625        let read_store = Arc::clone(&store);
10626        let read_opts = opts.clone();
10627        let read = tokio::spawn(async move {
10628            read_store
10629                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10630                .await
10631        });
10632        barrier.wait_after_fanout().await;
10633        *store.pool_meta.write().await = PoolMeta {
10634            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10635            ..Default::default()
10636        };
10637        barrier.release_after_fanout();
10638        barrier.wait_before_refetch().await;
10639
10640        let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10641        set_disks
10642            .put_object(bucket, object, &mut replacement_reader, &opts)
10643            .await
10644            .expect("replacement generation should be written before fallback refetch");
10645        barrier.release_before_refetch();
10646
10647        let error = match read.await.expect("prepared read task should not panic") {
10648            Ok(_) => panic!("changed fallback generation must not be accepted"),
10649            Err(error) => error,
10650        };
10651        assert_eq!(error, Error::ErasureReadQuorum);
10652    }
10653
10654    #[tokio::test]
10655    #[serial_test::serial(body_cache_hook)]
10656    async fn prepared_reader_rejects_latest_delete_marker_without_refetching_metadata() {
10657        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10658        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10659        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10660        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10661        let bucket = "prepared-reader-latest-delete-marker";
10662        let object = "versioned-object.bin";
10663        let versioned_opts = ObjectOptions {
10664            no_lock: true,
10665            versioned: true,
10666            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10667            ..Default::default()
10668        };
10669
10670        for set_disks in [&first_set, &second_set] {
10671            set_disks
10672                .make_bucket(bucket, &MakeBucketOptions::default())
10673                .await
10674                .expect("bucket should be created");
10675        }
10676        let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10677        first_set
10678            .put_object(bucket, object, &mut older_reader, &versioned_opts)
10679            .await
10680            .expect("older object should be written");
10681        let mut hidden_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10682        second_set
10683            .put_object(bucket, object, &mut hidden_reader, &versioned_opts)
10684            .await
10685            .expect("newer object should be written");
10686        let marker = second_set
10687            .delete_object(bucket, object, versioned_opts.clone())
10688            .await
10689            .expect("delete marker should be committed");
10690        assert!(marker.delete_marker);
10691
10692        let calls = disk_call_counters::observe(object);
10693        let error = match store
10694            .prepare_get_object_reader(
10695                bucket,
10696                object,
10697                None,
10698                HeaderMap::new(),
10699                &ObjectOptions {
10700                    no_lock: true,
10701                    versioned: true,
10702                    ..Default::default()
10703                },
10704            )
10705            .await
10706        {
10707            Ok(_) => panic!("latest delete marker should hide the older live object"),
10708            Err(error) => error,
10709        };
10710
10711        assert!(is_err_object_not_found(&error));
10712        assert!(
10713            calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10714            "delete-marker resolution must fan out each pool at most once"
10715        );
10716    }
10717
10718    #[tokio::test]
10719    #[serial_test::serial(body_cache_hook)]
10720    async fn prepared_reader_explicit_version_reuses_the_matching_pool_metadata() {
10721        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10722        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10723        let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await;
10724        let bucket = "prepared-reader-explicit-version";
10725        let object = "versioned-object.bin";
10726        let payload = b"explicit version from first pool".repeat(8_000);
10727        let versioned_opts = ObjectOptions {
10728            no_lock: true,
10729            versioned: true,
10730            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10731            ..Default::default()
10732        };
10733
10734        for set_disks in [&first_set, &second_set] {
10735            set_disks
10736                .make_bucket(bucket, &MakeBucketOptions::default())
10737                .await
10738                .expect("bucket should be created");
10739        }
10740        let mut first_reader = PutObjReader::from_vec(payload.clone());
10741        let first = first_set
10742            .put_object(bucket, object, &mut first_reader, &versioned_opts)
10743            .await
10744            .expect("requested version should be written to the first pool");
10745        let mut second_reader = PutObjReader::from_vec(b"different pool version".to_vec());
10746        second_set
10747            .put_object(bucket, object, &mut second_reader, &versioned_opts)
10748            .await
10749            .expect("a different version should be written to the second pool");
10750
10751        let requested_version = first
10752            .version_id
10753            .expect("versioned PUT should return a version id")
10754            .to_string();
10755        let read_opts = ObjectOptions {
10756            no_lock: true,
10757            versioned: true,
10758            version_id: Some(requested_version),
10759            ..Default::default()
10760        };
10761        let calls = disk_call_counters::observe(object);
10762        let prepared = store
10763            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10764            .await
10765            .expect("explicit version should resolve from the matching pool");
10766        assert_eq!(prepared.object_info().version_id, first.version_id);
10767        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10768        assert!(metadata_calls <= 8, "explicit-version lookup must fan out each pool at most once");
10769
10770        let mut reader = prepared
10771            .into_reader()
10772            .await
10773            .expect("prepared explicit-version body should open");
10774        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), metadata_calls);
10775        let mut restored = Vec::new();
10776        reader
10777            .stream
10778            .read_to_end(&mut restored)
10779            .await
10780            .expect("explicit-version body should stream");
10781        assert_eq!(restored, payload);
10782    }
10783
10784    #[tokio::test]
10785    #[serial_test::serial]
10786    async fn select_snapshot_locks_the_hashed_set_in_every_pool() {
10787        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10788        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10789        let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10790        let bucket = "select-snapshot-all-pools";
10791        let object = "object.bin";
10792        let payload = b"second-pool-snapshot".to_vec();
10793        let no_lock_opts = ObjectOptions {
10794            no_lock: true,
10795            ..Default::default()
10796        };
10797
10798        for pool in &store.pools {
10799            pool.make_bucket(bucket, &MakeBucketOptions::default())
10800                .await
10801                .expect("bucket should be created in each pool");
10802        }
10803        let mut put_reader = PutObjReader::from_vec(payload.clone());
10804        store.pools[1]
10805            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10806            .await
10807            .expect("object should be written only to the second pool");
10808
10809        let snapshot = store
10810            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10811            .await
10812            .expect("snapshot should resolve the second-pool object");
10813        assert_pool_writer_is_blocked(&store, 0, bucket, object).await;
10814        assert_pool_writer_is_blocked(&store, 1, bucket, object).await;
10815
10816        let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10817        let mut restored = Vec::new();
10818        reader
10819            .stream
10820            .read_to_end(&mut restored)
10821            .await
10822            .expect("snapshot body should stream");
10823        assert_eq!(restored, payload);
10824        drop(reader);
10825        drop(snapshot);
10826
10827        drop(acquire_pool_writer(&store, 0, bucket, object).await);
10828        drop(acquire_pool_writer(&store, 1, bucket, object).await);
10829    }
10830
10831    // Phase 5 Slice 2 (backlog#939): the instance context flows down the whole
10832    // object graph — ECStore, its Sets, and their SetDisks must all carry the
10833    // same `Arc<InstanceContext>` in a single-instance deployment.
10834    #[tokio::test]
10835    async fn instance_context_flows_through_object_graph() {
10836        let store = new_read_lock_test_store().await;
10837
10838        let sets = store.pools.first().expect("test store has one pool");
10839        assert!(
10840            std::sync::Arc::ptr_eq(&store.ctx, sets.instance_ctx()),
10841            "Sets must carry the store's instance context"
10842        );
10843
10844        let set_disks = sets.disk_set.first().expect("pool has one set");
10845        assert!(
10846            std::sync::Arc::ptr_eq(sets.instance_ctx(), set_disks.instance_ctx()),
10847            "SetDisks must carry the Sets' instance context"
10848        );
10849    }
10850
10851    // Phase 5 Slice 3 (backlog#939): a SetDisks sources its lock manager from
10852    // its instance context (not an independent process lookup), and in a
10853    // single-instance build that context aliases the process lock-manager
10854    // singleton — so the lock namespace is unchanged.
10855    #[tokio::test]
10856    async fn set_disks_lock_manager_comes_from_instance_context() {
10857        let store = new_read_lock_test_store().await;
10858        let set_disks = store.pools[0].disk_set.first().expect("pool has one set");
10859
10860        assert!(
10861            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &set_disks.instance_ctx().lock_manager()),
10862            "SetDisks lock manager must be sourced from its instance context"
10863        );
10864        assert!(
10865            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &rustfs_lock::get_global_lock_manager()),
10866            "single-instance lock manager must alias the process singleton"
10867        );
10868    }
10869
10870    #[tokio::test]
10871    #[serial_test::serial]
10872    async fn delete_objects_write_locks_cover_each_unique_object() {
10873        let store = new_read_lock_test_store().await;
10874        let objects = vec![
10875            ObjectToDelete {
10876                object_name: "beta".to_string(),
10877                ..Default::default()
10878            },
10879            ObjectToDelete {
10880                object_name: "alpha".to_string(),
10881                ..Default::default()
10882            },
10883            ObjectToDelete {
10884                object_name: "beta".to_string(),
10885                ..Default::default()
10886            },
10887        ];
10888        let mut opts = ObjectOptions::default();
10889
10890        let guards = store
10891            .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
10892            .await
10893            .expect("delete object locks should be acquired");
10894
10895        assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
10896        assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
10897        assert!(
10898            opts.namespace_lock_fence.is_some(),
10899            "set layer must receive the outer write-lock loss fence"
10900        );
10901
10902        let alpha_lock = store
10903            .handle_new_ns_lock("bucket", "alpha")
10904            .await
10905            .expect("alpha namespace lock should be created");
10906        let err = alpha_lock
10907            .get_read_lock(Duration::from_millis(20))
10908            .await
10909            .expect_err("batch delete write guard should block alpha readers");
10910        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
10911
10912        drop(guards);
10913        alpha_lock
10914            .get_read_lock(Duration::from_secs(1))
10915            .await
10916            .expect("alpha read lock should be available after dropping batch guards");
10917    }
10918
10919    #[tokio::test]
10920    async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
10921        let store = new_read_lock_test_store().await;
10922        let mut opts = ObjectOptions::default();
10923
10924        let guard = store
10925            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10926            .await
10927            .expect("read lock should be acquired");
10928
10929        assert!(guard.is_some(), "read lock should be held by the outer store layer");
10930        assert!(opts.no_lock, "set layer should not reacquire the object lock");
10931        assert!(
10932            opts.metadata_cache_safe,
10933            "metadata cache is safe only because the outer store layer acquired the read lock"
10934        );
10935    }
10936
10937    #[tokio::test]
10938    async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
10939        let store = new_read_lock_test_store().await;
10940        let mut opts = ObjectOptions {
10941            no_lock: true,
10942            ..Default::default()
10943        };
10944
10945        let guard = store
10946            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10947            .await
10948            .expect("prelocked read should not acquire another lock");
10949
10950        assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
10951        assert!(
10952            !opts.metadata_cache_safe,
10953            "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
10954        );
10955    }
10956
10957    // NOTE: #4877's `restore_transitioned_object_waits_for_existing_reader`
10958    // was removed with the whole-copy-back write lock it asserted
10959    // (backlog#1304): restore entry no longer serializes on the object lock.
10960    // The replacement semantics — non-blocking reads during the copy-back and
10961    // fast rejection of a concurrent restore — are covered end-to-end by
10962    // `restore_object_usecase_reports_ongoing_conflict`
10963    // (rustfs/src/app/lifecycle_transition_api_test.rs), while the SetDisks
10964    // transition matrix covers the final local commit. Restore-vs-reader data
10965    // protection lives in the inner put_object/complete_multipart_upload locks.
10966    #[tokio::test]
10967    #[serial_test::serial]
10968    async fn restore_accept_guard_serializes_concurrent_accepts() {
10969        // backlog#1304: the accept guard is the compare-and-set boundary for
10970        // the restore ongoing flag — a second accept of the same object must
10971        // wait behind (here: time out on) the first.
10972        let store = Arc::new(new_read_lock_test_store().await);
10973        let _first = store
10974            .acquire_restore_accept_guard("bucket", "object")
10975            .await
10976            .expect("first accept guard should be acquired");
10977
10978        let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
10979            match store.acquire_restore_accept_guard("bucket", "object").await {
10980                Ok(_) => panic!("second accept guard must wait behind the first"),
10981                Err(err) => err,
10982            }
10983        })
10984        .await;
10985
10986        assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
10987    }
10988
10989    #[tokio::test]
10990    #[serial_test::serial]
10991    async fn restore_worker_guard_proves_generation_liveness_until_drop() {
10992        let store = Arc::new(new_read_lock_test_store().await);
10993        let operation_id = Uuid::from_u128(1);
10994
10995        let first = store
10996            .acquire_restore_worker_guard(operation_id)
10997            .await
10998            .expect("first worker must own its newly generated lock");
10999        assert!(
11000            store
11001                .try_acquire_restore_worker_guard(operation_id)
11002                .await
11003                .expect("a contended liveness probe should remain typed")
11004                .is_none(),
11005            "the same generation must remain live while its worker guard is held"
11006        );
11007
11008        drop(first);
11009        assert!(
11010            store
11011                .try_acquire_restore_worker_guard(operation_id)
11012                .await
11013                .expect("post-drop liveness probe should not fail")
11014                .is_some(),
11015            "dropping the worker guard must make the orphan generation recoverable"
11016        );
11017    }
11018
11019    #[tokio::test]
11020    #[serial_test::serial]
11021    async fn reader_lock_is_held_when_optimization_is_disabled() {
11022        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
11023            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11024            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11025            let key = rustfs_lock::ObjectKey::new("bucket", "object");
11026            let read_guard = lock
11027                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11028                .await
11029                .expect("read lock should be acquired");
11030            let read_guard = ObjectLockDiagGuard::new(
11031                read_guard,
11032                true,
11033                "test_get_object",
11034                Some("bucket".to_string()),
11035                Some("object".to_string()),
11036                Some("reader".to_string()),
11037                ObjectLockDiagMode::Read,
11038            );
11039            let reader = GetObjectReader {
11040                stream: Box::new(Cursor::new(Vec::<u8>::new())),
11041                object_info: ObjectInfo::default(),
11042                buffered_body: None,
11043                body_source: Default::default(),
11044            };
11045
11046            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11047
11048            lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
11049                .await
11050                .expect_err("reader should hold the read lock");
11051            drop(reader);
11052            lock.get_write_lock(key, "writer", Duration::from_secs(1))
11053                .await
11054                .expect("dropping the reader should release the read lock");
11055        })
11056        .await;
11057    }
11058
11059    #[tokio::test]
11060    #[serial_test::serial]
11061    async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
11062        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
11063            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11064            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11065            let key = rustfs_lock::ObjectKey::new("bucket", "object");
11066            let read_guard = lock
11067                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11068                .await
11069                .expect("read lock should be acquired");
11070            let read_guard = ObjectLockDiagGuard::new(
11071                read_guard,
11072                true,
11073                "test_get_object",
11074                Some("bucket".to_string()),
11075                Some("object".to_string()),
11076                Some("reader".to_string()),
11077                ObjectLockDiagMode::Read,
11078            );
11079            let reader = GetObjectReader {
11080                stream: Box::new(Cursor::new(vec![1, 2, 3])),
11081                object_info: ObjectInfo::default(),
11082                buffered_body: None,
11083                body_source: Default::default(),
11084            };
11085
11086            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11087
11088            lock.get_write_lock(key, "writer", Duration::from_secs(1))
11089                .await
11090                .expect("lock optimization should release the read lock before returning the stream");
11091            drop(reader);
11092        })
11093        .await;
11094    }
11095
11096    #[tokio::test]
11097    #[serial_test::serial]
11098    async fn reader_lock_is_not_held_for_buffered_body_when_optimization_is_enabled() {
11099        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
11100            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11101            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11102            let key = rustfs_lock::ObjectKey::new("bucket", "object");
11103            let read_guard = lock
11104                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11105                .await
11106                .expect("read lock should be acquired");
11107            let read_guard = ObjectLockDiagGuard::new(
11108                read_guard,
11109                true,
11110                "test_get_object",
11111                Some("bucket".to_string()),
11112                Some("object".to_string()),
11113                Some("reader".to_string()),
11114                ObjectLockDiagMode::Read,
11115            );
11116            let reader = GetObjectReader {
11117                stream: Box::new(Cursor::new(vec![1, 2, 3])),
11118                object_info: ObjectInfo::default(),
11119                buffered_body: Some(Bytes::from_static(b"123")),
11120                body_source: Default::default(),
11121            };
11122
11123            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11124
11125            lock.get_write_lock(key, "writer", Duration::from_secs(1))
11126                .await
11127                .expect("buffered reader should release the read lock immediately");
11128            drop(reader);
11129        })
11130        .await;
11131    }
11132
11133    #[tokio::test]
11134    #[serial_test::serial]
11135    async fn reader_lock_is_released_after_stream_eof() {
11136        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
11137            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11138            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11139            let key = rustfs_lock::ObjectKey::new("bucket", "object");
11140            let read_guard = lock
11141                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11142                .await
11143                .expect("read lock should be acquired");
11144            let read_guard = ObjectLockDiagGuard::new(
11145                read_guard,
11146                true,
11147                "test_get_object",
11148                Some("bucket".to_string()),
11149                Some("object".to_string()),
11150                Some("reader".to_string()),
11151                ObjectLockDiagMode::Read,
11152            );
11153            let reader = GetObjectReader {
11154                stream: Box::new(Cursor::new(vec![1, 2, 3])),
11155                object_info: ObjectInfo::default(),
11156                buffered_body: None,
11157                body_source: Default::default(),
11158            };
11159
11160            let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11161            let mut output = Vec::new();
11162            reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
11163            assert_eq!(output, vec![1, 2, 3]);
11164
11165            lock.get_write_lock(key, "writer", Duration::from_secs(1))
11166                .await
11167                .expect("EOF should release the read lock before the reader is dropped");
11168            drop(reader);
11169        })
11170        .await;
11171    }
11172}