1use super::*;
16use crate::bucket::lifecycle::{
17 bucket_lifecycle_ops::{enqueue_committed_free_versions, eval_action_from_lifecycle},
18 get_expiry_configs,
19 tier_delete_journal::{
20 ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
21 TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
22 inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
23 record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
24 tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
25 },
26 tier_sweeper::{
27 Jentry, TierDeleteSourceIdentity, attach_tier_delete_source, transitioned_delete_journal_entry_for_source,
28 transitioned_force_delete_journal_entry,
29 },
30};
31use crate::bucket::metadata_sys::{
32 acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
33 get_object_lock_config_and_incarnation_from_disk_in,
34};
35use crate::bucket::object_lock::objectlock_sys::{
36 check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
37};
38use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
39use crate::bucket::versioning::VersioningApi;
40use crate::core::pools::{DecommissionCapacityAdmission, DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
41use crate::diagnostics::object_lock::ObjectLockAttempt;
42use crate::disk::OldCurrentSize;
43use crate::object_api::{
44 NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
45 TierFreeVersionReceiptSink,
46};
47use crate::services::notification_sys::{
48 TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
49 tier_delete_journal_topology_generation,
50};
51use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
52use crate::set_disk::{
53 SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
54 is_lock_optimization_enabled, is_object_lock_diag_enabled,
55};
56use crate::storage_api_contracts::{
57 list::ListOperations as _,
58 namespace::NamespaceLocking as _,
59 object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
60};
61use futures::StreamExt as _;
62use parking_lot::Mutex as ParkingMutex;
63use rustfs_filemeta::ObjectPartInfo;
64use rustfs_io_metrics::{
65 record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
66 record_object_lock_diag_slow_hold,
67};
68use std::{
69 fmt,
70 pin::Pin,
71 sync::atomic::{AtomicBool, Ordering},
72 task::{Context, Poll},
73 time::{Duration, Instant},
74};
75use tokio::io::{AsyncRead, ReadBuf};
76
77#[cfg(not(test))]
78const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
79#[cfg(test)]
80const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
81const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
82const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
83const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
84
85fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
86 if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
87 return None;
88 }
89
90 let sink = TierFreeVersionReceiptSink::new();
91 opts.tier_free_version_receipt_sink = Some(sink.clone());
92 Some(sink)
93}
94
95async fn enqueue_recorded_tier_free_versions(store: &ECStore, sink: Option<TierFreeVersionReceiptSink>) -> usize {
96 let Some(sink) = sink else {
97 return 0;
98 };
99 let Ok(receipts) = sink.drain() else {
100 return 0;
101 };
102 enqueue_committed_free_versions(store, receipts).await
103}
104
105fn build_tier_delete_journal_entry(
106 bucket: &str,
107 object: &str,
108 opts: &ObjectOptions,
109 source: &ObjectInfo,
110) -> Result<Option<Jentry>> {
111 let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
112 let source_object = decode_dir_object(object);
113 let entry = if opts.delete_prefix {
114 transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
115 attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
116 je
117 })
118 } else {
119 transitioned_delete_journal_entry_for_source(
120 version_id,
121 opts.versioned,
122 opts.version_suspended,
123 bucket,
124 source_object.as_str(),
125 source,
126 )
127 };
128 let Some(mut je) = entry else {
129 let lifecycle_exact_object_delete_all = opts.delete_prefix_object && opts.lifecycle_delete_all.is_some();
130 if opts.delete_prefix
131 && !lifecycle_exact_object_delete_all
132 && source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE
133 {
134 return Err(Error::other(
135 "recursive prefix delete cannot preserve a transitioned source with legacy Unknown remote-version state",
136 ));
137 }
138 return Ok(None);
139 };
140 record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
141 if opts.delete_prefix && !je.can_replace_tier_free_version() {
142 return Err(Error::other(
143 "recursive prefix delete requires a stable transitioned source and destination identity",
144 ));
145 }
146 Ok(Some(je))
147}
148
149async fn prepare_prefix_tier_delete_journal_entries(
150 api: &Arc<ECStore>,
151 bucket: &str,
152 prefix: &str,
153 opts: &ObjectOptions,
154) -> Result<PreparedPrefixTierDelete> {
155 Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
156}
157
158type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
159
160fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
161 truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
162}
163
164fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
165 let mut cancelled = false;
166 for result in results {
167 match result {
168 Ok(()) => {}
169 Err(Error::OperationCanceled) => cancelled = true,
170 Err(err) => return Err(err),
171 }
172 }
173 if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
174}
175
176async fn acquire_prefix_tier_delete_reference_leases(
177 api: &Arc<ECStore>,
178 tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
179) -> Result<Vec<TierOperationLease>> {
180 let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
181 tier_references.sort_unstable();
182 let mut leases = Vec::with_capacity(tier_references.len());
183 for (tier_name, backend_identity) in tier_references {
184 let lease = match backend_identity {
185 Some(backend_identity) => {
186 TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
187 .await
188 }
189 None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
190 }
191 .map_err(Error::other)?;
192 leases.push(lease);
193 }
194 Ok(leases)
195}
196
197async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
198 let tier_references = entries
199 .iter()
200 .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
201 .collect::<std::collections::HashSet<_>>();
202 acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
203}
204
205async fn prepare_prefix_tier_delete_journal_entries_inner(
206 api: &Arc<ECStore>,
207 bucket: &str,
208 prefix: &str,
209 opts: &ObjectOptions,
210) -> Result<PreparedPrefixTierDelete> {
211 let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
212 (false, false, None)
213 } else {
214 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
215 let bucket_fence = opts
216 .bucket_lifecycle_lock_fence
217 .as_ref()
218 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
219 match Box::pin(inspect_tier_delete_chunk_parent(
220 Arc::clone(api),
221 bucket,
222 bucket_incarnation,
223 prefix,
224 bucket_fence,
225 ))
226 .await?
227 {
228 TierDeleteChunkParentInspection::NoParent => (false, false, None),
229 TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
230 TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
231 TierDeleteChunkParentInspection::Resume(dispatch) => {
232 let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
233 return Ok(PreparedPrefixTierDelete {
234 dispatch: Some(*dispatch),
235 chunk_parent_active: true,
236 chunk_parent_fleet_proof: None,
237 _leases: leases,
238 });
239 }
240 TierDeleteChunkParentInspection::RetryRequired => {
241 return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
242 }
243 }
244 };
245 let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
246 let mut entries_by_name = std::collections::BTreeMap::new();
247 let batch_limit = tier_delete_dispatch_batch_limit();
248 let logical_prefix = decode_dir_object(prefix);
249 let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
250 let physical_sets = api
251 .pools
252 .iter()
253 .flat_map(|pool| pool.disk_set.iter().cloned())
254 .collect::<Vec<_>>();
255 let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
256 let cancellation = tokio_util::sync::CancellationToken::new();
257 let limit_cancellation = Arc::new(AtomicBool::new(false));
258 let walk_cancel = cancellation.clone();
259 let bucket_owned = bucket.to_string();
260 let prefix_owned = prefix.to_string();
261 let walk = async move {
262 let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
263 let tx = tx.clone();
264 let cancellation = walk_cancel.clone();
265 let bucket = bucket_owned.clone();
266 let prefix = prefix_owned.clone();
267 async move {
268 let result = set
269 .walk(
270 cancellation.clone(),
271 &bucket,
272 &prefix,
273 tx,
274 WalkOptions {
275 include_free_versions: true,
276 walkdir_timeout: Some(Duration::ZERO),
277 ..Default::default()
278 },
279 )
280 .await;
281 if result.is_err() {
282 cancellation.cancel();
283 }
284 result
285 }
286 }))
287 .buffer_unordered(4)
288 .collect::<Vec<_>>()
289 .await;
290 drop(tx);
291 combine_tier_delete_walk_results(results)
292 };
293 let collect_limit_cancellation = limit_cancellation.clone();
294 let collect = async {
295 let mut truncated = false;
296 while let Some(result) = rx.recv().await {
297 if let Some(err) = result.err {
298 if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
304 {
305 continue;
306 }
307 cancellation.cancel();
308 return Err(err);
309 }
310 if truncated {
311 continue;
312 }
313 let Some(source) = result.item else {
314 continue;
315 };
316 let object = decode_dir_object(&source.name);
317 if exact_object.as_ref().is_some_and(|expected| expected != &object) {
318 continue;
319 }
320 if exact_object.is_none() && !object.starts_with(&logical_prefix) {
321 continue;
322 }
323 if source.transitioned_object.free_version {
324 cancellation.cancel();
325 return Err(Error::other(
326 "recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
327 ));
328 }
329 let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
330 let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
331 Some((source.transitioned_object.tier.clone(), backend_identity))
332 } else {
333 None
334 };
335 if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
336 let name = tier_delete_journal_object_name(&entry);
337 let at_limit = entries_by_name.len() == batch_limit;
338 match entries_by_name.entry(name) {
339 std::collections::btree_map::Entry::Occupied(_) => {}
340 std::collections::btree_map::Entry::Vacant(_) if at_limit => {
341 truncated = true;
342 collect_limit_cancellation.store(true, Ordering::Release);
343 cancellation.cancel();
344 }
345 std::collections::btree_map::Entry::Vacant(slot) => {
346 if let Some(tier_reference) = tier_reference {
347 tier_references.insert(tier_reference);
348 }
349 slot.insert(entry);
350 }
351 }
352 } else if let Some(tier_reference) = tier_reference {
353 tier_references.insert(tier_reference);
354 }
355 }
356 Ok(truncated)
357 };
358 let (walk_result, collect_result) = tokio::join!(walk, collect);
359 let truncated = collect_result?;
360 if let Err(err) = walk_result
364 && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
365 {
366 return Err(err);
367 }
368 let entries = entries_by_name.into_values().collect::<Vec<_>>();
369 let mut leased_tier_references = tier_references;
370 let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
371 if entries.is_empty() {
372 let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
373 let fleet_proof = acquire_tier_delete_journal_fleet_proof()
374 .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
375 if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
376 return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
377 }
378 Some(fleet_proof)
379 } else {
380 None
381 };
382 return Ok(PreparedPrefixTierDelete {
383 dispatch: None,
384 chunk_parent_active,
385 chunk_parent_fleet_proof,
386 _leases: leases,
387 });
388 }
389
390 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
391 let bucket_fence = opts
392 .bucket_lifecycle_lock_fence
393 .as_ref()
394 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
395 let fleet_proof = acquire_tier_delete_journal_fleet_proof()
396 .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
397 if chunk_parent_topology_generation
398 .as_deref()
399 .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
400 {
401 return Err(Error::other("tier delete chunk parent topology changed during source scan"));
402 }
403 let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
404 Box::pin(prepare_tier_delete_chunk_dispatch(
405 Arc::clone(api),
406 bucket,
407 bucket_incarnation,
408 prefix,
409 entries,
410 truncated && !chunk_parent_active,
411 fleet_proof,
412 bucket_fence,
413 ))
414 .await?
415 } else if legacy_manifest_active && truncated {
416 resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
417 .await?
418 } else {
419 prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
420 .await?
421 };
422 let additional_tier_references = dispatch
427 .entries()?
428 .iter()
429 .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
430 .filter(|reference| leased_tier_references.insert(reference.clone()))
431 .collect::<std::collections::HashSet<_>>();
432 leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
433 if legacy_manifest_active && truncated {
434 dispatch.require_exact_predecessor_replay();
435 }
436 Ok(PreparedPrefixTierDelete {
437 dispatch: Some(dispatch),
438 chunk_parent_active: chunk_parent_active || truncated,
439 chunk_parent_fleet_proof: None,
440 _leases: leases,
441 })
442}
443
444struct PreparedPrefixTierDelete {
445 dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
446 chunk_parent_active: bool,
447 chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
448 _leases: Vec<TierOperationLease>,
449}
450
451#[cfg(all(test, feature = "test-util"))]
452struct PrefixDeleteAfterJournalPrepareBarrierState {
453 bucket: String,
454 prefix: String,
455 arrived: tokio::sync::Notify,
456 release: tokio::sync::Notify,
457}
458
459#[cfg(all(test, feature = "test-util"))]
460pub(crate) struct PrefixDeleteAfterJournalPrepareBarrier {
461 state: Arc<PrefixDeleteAfterJournalPrepareBarrierState>,
462}
463
464#[cfg(all(test, feature = "test-util"))]
465static PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER: std::sync::OnceLock<
466 std::sync::Mutex<Option<Arc<PrefixDeleteAfterJournalPrepareBarrierState>>>,
467> = std::sync::OnceLock::new();
468
469#[cfg(all(test, feature = "test-util"))]
470impl PrefixDeleteAfterJournalPrepareBarrier {
471 pub(crate) fn install(bucket: &str, prefix: &str) -> Self {
472 let state = Arc::new(PrefixDeleteAfterJournalPrepareBarrierState {
473 bucket: bucket.to_string(),
474 prefix: prefix.to_string(),
475 arrived: tokio::sync::Notify::new(),
476 release: tokio::sync::Notify::new(),
477 });
478 let mut slot = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
479 .get_or_init(|| std::sync::Mutex::new(None))
480 .lock()
481 .expect("prefix journal barrier mutex should not poison");
482 assert!(slot.is_none(), "prefix journal barrier must not already be installed");
483 *slot = Some(Arc::clone(&state));
484 Self { state }
485 }
486
487 pub(crate) async fn wait_until_paused(&self) {
488 self.state.arrived.notified().await;
489 }
490
491 pub(crate) fn release(&self) {
492 self.state.release.notify_one();
493 }
494}
495
496#[cfg(all(test, feature = "test-util"))]
497impl Drop for PrefixDeleteAfterJournalPrepareBarrier {
498 fn drop(&mut self) {
499 self.state.release.notify_one();
500 if let Some(slot) = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER.get() {
501 let mut slot = slot.lock().expect("prefix journal barrier mutex should not poison");
502 if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
503 *slot = None;
504 }
505 }
506 }
507}
508
509#[cfg(all(test, feature = "test-util"))]
510async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) {
511 let state = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
512 .get_or_init(|| std::sync::Mutex::new(None))
513 .lock()
514 .expect("prefix journal barrier mutex should not poison")
515 .as_ref()
516 .filter(|state| state.bucket == bucket && state.prefix == prefix)
517 .cloned();
518 if let Some(state) = state {
519 state.arrived.notify_one();
520 state.release.notified().await;
521 }
522}
523
524async fn delete_prefix_with_tier_delete_journal(
525 store: &ECStore,
526 bucket: &str,
527 object: &str,
528 opts: &ObjectOptions,
529 tier_journal_api: Option<&Arc<ECStore>>,
530) -> Result<()> {
531 let Some(api) = tier_journal_api else {
532 return store.delete_prefix(bucket, object, opts).await;
533 };
534 let PreparedPrefixTierDelete {
535 dispatch,
536 chunk_parent_active,
537 chunk_parent_fleet_proof,
538 _leases,
539 } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
540 let Some(dispatch) = dispatch else {
541 let _tier_leases = _leases;
545 let mut operation_opts = opts.clone();
546 operation_opts.tier_delete_journal_api = None;
551 operation_opts.tier_delete_dispatch_authorization = None;
552 let parent_fleet_proof = if chunk_parent_active {
553 Some(
554 chunk_parent_fleet_proof
555 .as_ref()
556 .filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
557 .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
558 )
559 } else {
560 None
561 };
562 store.delete_prefix(bucket, object, &operation_opts).await?;
563 if let Some(parent_fleet_proof) = parent_fleet_proof {
564 #[cfg(all(test, feature = "test-util"))]
565 crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
566 crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
567 )
568 .await;
569 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
570 let bucket_fence = opts
571 .bucket_lifecycle_lock_fence
572 .as_ref()
573 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
574 if !Box::pin(complete_tier_delete_chunk_parent(
575 Arc::clone(api),
576 bucket,
577 bucket_incarnation,
578 object,
579 bucket_fence,
580 parent_fleet_proof,
581 ))
582 .await?
583 {
584 return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
585 }
586 }
587 return Ok(());
588 };
589 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
590 let bucket_fence = opts
591 .bucket_lifecycle_lock_fence
592 .as_ref()
593 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
594 let active = dispatch.consume(bucket, bucket_incarnation, object)?;
595 let mut operation_opts = opts.clone();
596 operation_opts.tier_delete_dispatch_authorization = Some(active.authorization());
597
598 #[cfg(all(test, feature = "test-util"))]
599 pause_prefix_delete_after_journal_prepare(bucket, object).await;
600
601 let _tier_leases = _leases;
604 if active.predecessor_replay_required() || active.is_chunked() {
605 replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
606 #[cfg(all(test, feature = "test-util"))]
607 if active.is_chunked() {
608 crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
609 crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
610 )
611 .await;
612 }
613 complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
614 if active.is_chunked() {
615 return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
616 }
617 return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
618 }
619 let result = store.delete_prefix(bucket, object, &operation_opts).await;
620 match result {
621 Ok(()) => {
622 if let Err(err) = complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await {
623 warn!(
624 event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
625 component = LOG_COMPONENT_ECSTORE,
626 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
627 bucket,
628 prefix = object,
629 error = ?err,
630 "Prefix deletion committed with durable tier cleanup pending"
631 );
632 }
633 Ok(())
634 }
635 Err(err) => Err(err),
638 }
639}
640
641async fn replay_authorized_tier_delete_sources(
642 store: &ECStore,
643 bucket: &str,
644 prefix: &str,
645 active: &ActiveTierDeleteDispatch,
646 opts: &ObjectOptions,
647) -> Result<()> {
648 let _publication_scope_guard = opts
649 .scanner_publication_commit_scope
650 .clone()
651 .map(ScannerPublicationCommitScopeGuard::new);
652 let publication_scope = opts.scanner_publication_commit_scope.as_ref();
653 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
654 let bucket_fence = opts
655 .bucket_lifecycle_lock_fence
656 .as_ref()
657 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
658 let authorization = active.authorization();
659 authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
660
661 let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
662 for entry in active.entries() {
663 let source = entry
664 .source
665 .as_ref()
666 .filter(|source| source.has_stable_identity())
667 .ok_or_else(|| Error::other("authorized tier delete predecessor has no stable source"))?;
668 if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
669 return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
670 }
671 source_objects.insert(source.object.as_str());
672 }
673
674 if let Some(scope) = publication_scope {
675 if scope.state() == ScannerPublicationCommitState::Admitted {
676 scope
677 .try_begin()
678 .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
679 }
680 if !scope.can_commit() {
681 let _ = scope.mark_indeterminate();
682 return Err(StorageError::OperationCanceled);
683 }
684 }
685
686 let stopped = Arc::new(AtomicBool::new(false));
687 let make_replay = |object: String| {
691 let stopped = stopped.clone();
692 let authorization = authorization.clone();
693 async move {
694 if stopped.load(Ordering::Acquire) {
695 return Ok::<_, Error>(0usize);
696 }
697 let result = async {
698 if bucket_fence.is_lock_lost() {
699 return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
700 }
701 let encoded_object = encode_dir_object(&object);
702 let guards = if opts.delete_prefix_object {
703 store
704 .acquire_remaining_physical_object_write_locks(
705 "tier_delete_dispatch_predecessor_replay",
706 bucket,
707 &encoded_object,
708 )
709 .await?
710 } else {
711 store
712 .acquire_all_physical_object_write_locks(
713 "tier_delete_dispatch_predecessor_replay",
714 bucket,
715 &encoded_object,
716 )
717 .await?
718 };
719 authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
720 if publication_scope.is_some_and(|scope| !scope.can_commit()) {
721 return Err(StorageError::OperationCanceled);
722 }
723 let mut replay_opts = opts.clone();
724 replay_opts.no_lock = true;
725 replay_opts.delete_prefix = false;
726 replay_opts.delete_prefix_object = false;
727 for guard in &guards {
728 guard.add_namespace_lock_fence(&mut replay_opts);
729 }
730 let mut deleted = 0usize;
731 for pool in &store.pools {
732 for set in &pool.disk_set {
733 authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
734 deleted = deleted
735 .checked_add(
736 set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
737 .await?,
738 )
739 .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
740 }
741 }
742 if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
743 return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
744 }
745 Ok(deleted)
746 }
747 .await;
748 if result.is_err() {
749 stopped.store(true, Ordering::Release);
750 }
751 result
752 }
753 };
754 let mut objects = source_objects.into_iter();
755 let mut replays = futures::stream::FuturesUnordered::new();
756 for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
757 let Some(object) = objects.next().map(ToOwned::to_owned) else {
758 break;
759 };
760 replays.push(make_replay(object));
761 }
762 let mut deleted = 0usize;
763 let mut first_error = None;
764 while let Some(result) = replays.next().await {
765 match result {
766 Ok(count) => {
767 deleted = deleted
768 .checked_add(count)
769 .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
770 }
771 Err(err) if first_error.is_none() => first_error = Some(err),
772 Err(_) => {}
773 }
774 if first_error.is_none()
775 && !stopped.load(Ordering::Acquire)
776 && let Some(object) = objects.next().map(ToOwned::to_owned)
777 {
778 replays.push(make_replay(object));
779 }
780 }
781 if deleted > 0 {
782 super::list_objects::observe_list_objects_mutation(store, bucket).await;
783 }
784 if let Some(err) = first_error {
785 if publication_scope.is_some_and(|scope| !scope.can_commit())
786 && let Some(scope) = publication_scope
787 {
788 let _ = scope.mark_indeterminate();
789 }
790 return Err(err);
791 }
792 if let Some(scope) = publication_scope {
793 let _ = scope.mark_committed();
794 }
795 Ok(())
796}
797
798fn tier_delete_source_matches_replay_scope(
799 source: &TierDeleteSourceIdentity,
800 bucket: &str,
801 persisted_prefix: &str,
802 exact_object: bool,
803) -> bool {
804 tier_delete_source_matches_dispatch_scope(source, bucket, persisted_prefix)
805 && (!exact_object || source.object == decode_dir_object(persisted_prefix))
806}
807
808async fn delete_recursive_prefix_with_tier_delete_journal(
809 store: &ECStore,
810 bucket: &str,
811 object: &str,
812 opts: &ObjectOptions,
813 tier_journal_api: Option<&Arc<ECStore>>,
814) -> Result<()> {
815 if !is_meta_bucketname(bucket) {
818 let state = opts
819 .object_lock_config_snapshot
820 .as_deref()
821 .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))?
822 .state();
823 ensure_recursive_force_delete_allowed_for_state(bucket, state)?;
824 let bypass_governance = opts
825 .object_lock_delete
826 .as_ref()
827 .is_some_and(|delete_opts| delete_opts.bypass_governance);
828 for pool in &store.pools {
829 for set in &pool.disk_set {
830 let mut marker = None;
831 let mut version_marker = None;
832 loop {
833 let page = set
834 .clone()
835 .inner_list_object_versions_for_recursive_delete(
836 bucket,
837 object,
838 marker.clone(),
839 version_marker.clone(),
840 RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
841 )
842 .await?;
843 for object_info in &page.objects {
844 if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() {
845 return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone()));
846 }
847 }
848 if !page.is_truncated {
849 break;
850 }
851 let next_marker = page
852 .next_marker
853 .ok_or_else(|| Error::other("recursive delete version scan did not return a continuation marker"))?;
854 if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker {
855 return Err(Error::other("recursive delete version scan did not advance"));
856 }
857 marker = Some(next_marker);
858 version_marker = page.next_version_idmarker;
859 }
860 }
861 }
862 }
863 if object.ends_with('/') && !is_meta_bucketname(bucket) {
868 let parent = object.strip_suffix('/').unwrap_or(object);
869 for pool in &store.pools {
870 for set in &pool.disk_set {
871 let page = set
872 .clone()
873 .inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
874 .await?;
875 if page.objects.iter().any(|info| info.name == parent) {
876 return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
877 }
878 }
879 }
880 }
881 delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
882}
883
884async fn delete_directory_keys_with_tier_delete_journal(
885 store: &ECStore,
886 bucket: &str,
887 prefix: &str,
888 opts: &ObjectOptions,
889 tier_journal_api: Option<&Arc<ECStore>>,
890) -> Result<()> {
891 for pool in &store.pools {
892 for set in &pool.disk_set {
893 let mut previous_keys = std::collections::BTreeSet::new();
894 loop {
895 let page = set
898 .clone()
899 .inner_list_object_versions_for_recursive_delete(
900 bucket,
901 prefix,
902 None,
903 None,
904 RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
905 )
906 .await?;
907 let keys = page
908 .objects
909 .into_iter()
910 .map(|info| info.name)
911 .filter(|key| key.starts_with(prefix))
912 .collect::<std::collections::BTreeSet<_>>();
913 if keys.is_empty() {
914 break;
915 }
916 if keys == previous_keys {
917 return Err(Error::other("directory deletion did not advance"));
918 }
919 for key in &keys {
920 let encoded_key = encode_dir_object(key);
921 let mut exact_opts = opts.clone();
922 exact_opts.delete_prefix_object = true;
923 let _guard = store
924 .acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
925 .await?;
926 delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
927 }
928 previous_keys = keys;
929 }
930 }
931 }
932 Ok(())
933}
934
935pub struct PreparedGetObjectReader {
942 pool: Arc<Sets>,
943 bucket: String,
944 object: String,
945 range: Option<HTTPRangeSpec>,
946 headers: HeaderMap,
947 opts: ObjectOptions,
948 metadata: crate::set_disk::PreparedGetObjectMetadata,
949 read_lock_guard: Option<ObjectLockDiagGuard>,
950}
951
952impl PreparedGetObjectReader {
953 pub fn object_info(&self) -> &ObjectInfo {
955 self.metadata.object_info()
956 }
957
958 pub fn into_object_info(mut self) -> ObjectInfo {
960 self.metadata.take_object_info()
961 }
962
963 #[must_use]
965 pub fn with_headers(mut self, headers: HeaderMap) -> Self {
966 self.headers = headers;
967 self
968 }
969
970 pub async fn into_reader(self) -> Result<GetObjectReader> {
975 let mut reader =
976 crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
977 &self.bucket,
978 &self.object,
979 self.range,
980 self.headers,
981 &self.opts,
982 self.metadata,
983 ))
984 .await?;
985 reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
986 Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
987 }
988}
989
990struct LockGuardedReader {
991 inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
992 guard: Option<ObjectLockDiagGuard>,
993}
994
995impl AsyncRead for LockGuardedReader {
996 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
997 let had_capacity = buf.remaining() > 0;
998 let filled_before = buf.filled().len();
999 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1000 if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
1001 self.guard.take();
1002 }
1003 poll
1004 }
1005}
1006
1007#[derive(Clone, Copy, Debug)]
1008enum ObjectLockDiagMode {
1009 Read,
1010 Write,
1011}
1012
1013impl ObjectLockDiagMode {
1014 fn as_str(self) -> &'static str {
1015 match self {
1016 Self::Read => "read",
1017 Self::Write => "write",
1018 }
1019 }
1020}
1021
1022impl fmt::Display for ObjectLockDiagMode {
1023 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1024 f.write_str(self.as_str())
1025 }
1026}
1027
1028pub(crate) struct ObjectLockDiagGuard {
1029 guard: rustfs_lock::NamespaceLockGuard,
1030 #[cfg(test)]
1031 test_namespace_lock_fence: Option<NamespaceLockFence>,
1032 enabled: bool,
1033 op: &'static str,
1034 bucket: Option<String>,
1035 object: Option<String>,
1036 owner: Option<String>,
1037 mode: ObjectLockDiagMode,
1038 acquired_at: Instant,
1039 _attempt: ObjectLockAttempt,
1040}
1041
1042impl ObjectLockDiagGuard {
1043 fn new(
1044 guard: rustfs_lock::NamespaceLockGuard,
1045 enabled: bool,
1046 op: &'static str,
1047 bucket: Option<String>,
1048 object: Option<String>,
1049 owner: Option<String>,
1050 mode: ObjectLockDiagMode,
1051 ) -> Self {
1052 Self {
1053 guard,
1054 #[cfg(test)]
1055 test_namespace_lock_fence: None,
1056 enabled,
1057 op,
1058 bucket,
1059 object,
1060 owner,
1061 mode,
1062 acquired_at: Instant::now(),
1063 _attempt: ObjectLockAttempt::default(),
1064 }
1065 }
1066
1067 fn with_attempt(mut self, attempt: ObjectLockAttempt) -> Self {
1068 self._attempt = attempt;
1069 self
1070 }
1071
1072 pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
1073 match &self.guard {
1074 rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
1075 rustfs_lock::NamespaceLockGuard::Fast(_) => None,
1076 }
1077 }
1078
1079 pub(crate) fn is_lock_lost(&self) -> bool {
1080 self.guard.is_lock_lost() || {
1081 #[cfg(test)]
1082 {
1083 self.test_namespace_lock_fence
1084 .as_ref()
1085 .is_some_and(NamespaceLockFence::is_lock_lost)
1086 }
1087 #[cfg(not(test))]
1088 {
1089 false
1090 }
1091 }
1092 }
1093
1094 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1095 opts.ensure_namespace_lock_fence();
1096 if let Some(signal) = self.lock_lost_signal() {
1097 opts.add_namespace_lock_lost_signal(signal);
1098 }
1099 #[cfg(test)]
1100 if let Some(fence) = self.test_namespace_lock_fence.as_ref() {
1101 opts.add_namespace_lock_fence(fence);
1102 }
1103 }
1104}
1105
1106#[cfg(test)]
1107#[derive(Clone, Copy, PartialEq, Eq)]
1108pub(crate) enum DecommissionMutationFenceTestPhase {
1109 Migration,
1110 SourceCleanup,
1111}
1112
1113#[cfg(test)]
1114struct DecommissionMutationFenceLossState {
1115 bucket: String,
1116 object: String,
1117 phase: DecommissionMutationFenceTestPhase,
1118 fence: NamespaceLockFence,
1119 loss_handle: Arc<std::sync::atomic::AtomicBool>,
1120}
1121
1122#[cfg(test)]
1123pub(crate) struct DecommissionMutationFenceLossHook {
1124 state: Arc<DecommissionMutationFenceLossState>,
1125}
1126
1127#[cfg(test)]
1128static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock<
1129 std::sync::Mutex<Option<Arc<DecommissionMutationFenceLossState>>>,
1130> = std::sync::OnceLock::new();
1131
1132#[cfg(test)]
1133impl DecommissionMutationFenceLossHook {
1134 pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self {
1135 let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
1136 let state = Arc::new(DecommissionMutationFenceLossState {
1137 bucket: bucket.to_string(),
1138 object: object.to_string(),
1139 phase,
1140 fence,
1141 loss_handle,
1142 });
1143 let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1144 .get_or_init(|| std::sync::Mutex::new(None))
1145 .lock()
1146 .expect("decommission mutation fence loss hooks should not poison");
1147 assert!(slot.is_none(), "decommission mutation fence loss hook must be unique");
1148 *slot = Some(Arc::clone(&state));
1149 Self { state }
1150 }
1151
1152 pub(crate) fn mark_lost(&self) {
1153 self.state.loss_handle.store(true, Ordering::Release);
1154 }
1155}
1156
1157#[cfg(test)]
1158impl Drop for DecommissionMutationFenceLossHook {
1159 fn drop(&mut self) {
1160 let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1161 .get_or_init(|| std::sync::Mutex::new(None))
1162 .lock()
1163 .expect("decommission mutation fence loss hooks should not poison");
1164 if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) {
1165 *slot = None;
1166 }
1167 }
1168}
1169
1170#[cfg(test)]
1171fn decommission_mutation_fence_for_test(
1172 bucket: &str,
1173 object: &str,
1174 phase: DecommissionMutationFenceTestPhase,
1175) -> Option<NamespaceLockFence> {
1176 DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1177 .get_or_init(|| std::sync::Mutex::new(None))
1178 .lock()
1179 .expect("decommission mutation fence loss hooks should not poison")
1180 .as_ref()
1181 .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase)
1182 .map(|hook| hook.fence.clone())
1183}
1184
1185#[cfg(all(test, feature = "test-util"))]
1186struct DecommissionFreeVersionSourceRaceState {
1187 bucket: String,
1188 object: String,
1189 arrived: tokio::sync::Notify,
1190 release: tokio::sync::Notify,
1191}
1192
1193#[cfg(all(test, feature = "test-util"))]
1194pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
1195 state: Arc<DecommissionFreeVersionSourceRaceState>,
1196}
1197
1198#[cfg(all(test, feature = "test-util"))]
1199static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
1200 std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
1201> = std::sync::OnceLock::new();
1202
1203#[cfg(all(test, feature = "test-util"))]
1204impl DecommissionFreeVersionSourceRaceBarrier {
1205 pub(crate) fn install(bucket: &str, object: &str) -> Self {
1206 let state = Arc::new(DecommissionFreeVersionSourceRaceState {
1207 bucket: bucket.to_string(),
1208 object: object.to_string(),
1209 arrived: tokio::sync::Notify::new(),
1210 release: tokio::sync::Notify::new(),
1211 });
1212 let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1213 .get_or_init(|| std::sync::Mutex::new(None))
1214 .lock()
1215 .expect("decommission free-version source race barrier should not poison");
1216 assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
1217 *slot = Some(Arc::clone(&state));
1218 Self { state }
1219 }
1220
1221 pub(crate) async fn wait_until_paused(&self) {
1222 tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1223 .await
1224 .expect("decommission should pause before acquiring the free-version source lock");
1225 }
1226
1227 pub(crate) fn release(&self) {
1228 self.state.release.notify_one();
1229 }
1230}
1231
1232#[cfg(all(test, feature = "test-util"))]
1233impl Drop for DecommissionFreeVersionSourceRaceBarrier {
1234 fn drop(&mut self) {
1235 self.state.release.notify_one();
1236 let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1237 .get_or_init(|| std::sync::Mutex::new(None))
1238 .lock()
1239 .expect("decommission free-version source race barrier should not poison");
1240 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1241 *slot = None;
1242 }
1243 }
1244}
1245
1246#[cfg(all(test, feature = "test-util"))]
1247async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
1248 let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1249 .get_or_init(|| std::sync::Mutex::new(None))
1250 .lock()
1251 .expect("decommission free-version source race barrier should not poison")
1252 .as_ref()
1253 .filter(|state| state.bucket == bucket && state.object == object)
1254 .cloned();
1255 if let Some(state) = state {
1256 state.arrived.notify_one();
1257 state.release.notified().await;
1258 }
1259}
1260
1261pub(crate) struct SourceCleanupMutationFence {
1262 guard: ObjectLockDiagGuard,
1263 source_lock_covered: bool,
1264}
1265
1266impl SourceCleanupMutationFence {
1267 pub(crate) fn source_lock_covered(&self) -> bool {
1268 self.source_lock_covered
1269 }
1270
1271 pub(crate) fn is_lock_lost(&self) -> bool {
1272 self.guard.is_lock_lost()
1273 }
1274
1275 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1276 self.guard.add_namespace_lock_fence(opts);
1277 }
1278}
1279
1280pub struct RestoreAcceptGuard(ObjectLockDiagGuard);
1285
1286impl RestoreAcceptGuard {
1287 pub fn is_lock_lost(&self) -> bool {
1291 self.0.guard.is_lock_lost()
1292 }
1293
1294 pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1295 self.0.add_namespace_lock_fence(opts);
1296 }
1297}
1298
1299pub struct RestoreWorkerGuard(rustfs_lock::NamespaceLockGuard);
1303
1304impl RestoreWorkerGuard {
1305 pub fn is_lock_lost(&self) -> bool {
1306 self.0.is_lock_lost()
1307 }
1308}
1309
1310impl Drop for ObjectLockDiagGuard {
1311 fn drop(&mut self) {
1312 if !self.enabled || self.guard.is_released() {
1313 return;
1314 }
1315
1316 let hold = self.acquired_at.elapsed();
1317 record_object_lock_diag_hold_duration(self.op, self.mode.as_str(), hold);
1318 let threshold = get_object_lock_diag_slow_hold_threshold();
1319 if hold >= threshold {
1320 record_object_lock_diag_slow_hold(self.op, self.mode.as_str());
1321 warn!(
1322 target: "rustfs_ecstore::object_lock_diag",
1323 op = self.op,
1324 bucket = %self.bucket.as_deref().unwrap_or_default(),
1325 object = %self.object.as_deref().unwrap_or_default(),
1326 mode = %self.mode,
1327 owner = %self.owner.as_deref().unwrap_or_default(),
1328 hold_ms = hold.as_millis(),
1329 threshold_ms = threshold.as_millis(),
1330 "object namespace lock held longer than threshold"
1331 );
1332 }
1333 }
1334}
1335
1336#[derive(Debug, thiserror::Error)]
1338#[non_exhaustive]
1339pub enum SnapshotConsistencyError {
1340 #[error("namespace locking is disabled for SelectObjectContent")]
1341 LockingDisabled,
1342 #[error("SelectObjectContent namespace lock was lost")]
1343 LockLost,
1344 #[error("object read semantics changed while SelectObjectContent was running")]
1345 ObjectChanged,
1346}
1347
1348#[derive(Debug, thiserror::Error)]
1350#[non_exhaustive]
1351pub enum PrepareSelectObjectSnapshotError {
1352 #[error("storage failed while preparing SelectObjectContent snapshot: {0}")]
1353 Storage(#[source] StorageError),
1354 #[error("SelectObjectContent snapshot consistency failure: {0}")]
1355 Consistency(#[source] SnapshotConsistencyError),
1356 #[error("SelectObjectContent object has invalid logical size {size}")]
1357 InvalidLogicalSize { size: i64 },
1358}
1359
1360impl From<StorageError> for PrepareSelectObjectSnapshotError {
1361 fn from(error: StorageError) -> Self {
1362 Self::Storage(error)
1363 }
1364}
1365
1366impl From<SnapshotConsistencyError> for PrepareSelectObjectSnapshotError {
1367 fn from(error: SnapshotConsistencyError) -> Self {
1368 Self::Consistency(error)
1369 }
1370}
1371
1372#[derive(Debug, thiserror::Error)]
1374#[non_exhaustive]
1375pub enum SelectObjectSnapshotReadError {
1376 #[error("storage failed while opening SelectObjectContent snapshot reader: {0}")]
1377 Storage(#[source] StorageError),
1378 #[error("SelectObjectContent snapshot consistency failure: {0}")]
1379 Consistency(#[source] SnapshotConsistencyError),
1380}
1381
1382impl From<StorageError> for SelectObjectSnapshotReadError {
1383 fn from(error: StorageError) -> Self {
1384 Self::Storage(error)
1385 }
1386}
1387
1388impl From<SnapshotConsistencyError> for SelectObjectSnapshotReadError {
1389 fn from(error: SnapshotConsistencyError) -> Self {
1390 Self::Consistency(error)
1391 }
1392}
1393
1394struct SelectObjectSnapshotLease {
1395 guards: Vec<ObjectLockDiagGuard>,
1396 lost: Arc<AtomicBool>,
1397 lock_loss: tokio::sync::watch::Sender<bool>,
1398 _monitor_shutdown: tokio::sync::watch::Sender<()>,
1399}
1400
1401impl SelectObjectSnapshotLease {
1402 fn new(guards: Vec<ObjectLockDiagGuard>) -> Self {
1403 let signals = guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal);
1404 let mut waits = signals
1405 .map(|signal| async move { signal.notified().await })
1406 .collect::<futures::stream::FuturesUnordered<_>>();
1407 let lost = Arc::new(AtomicBool::new(false));
1408 let (lock_loss, _) = tokio::sync::watch::channel(false);
1409 let (monitor_shutdown, mut shutdown_rx) = tokio::sync::watch::channel(());
1410 if !waits.is_empty() {
1411 let task_lost = Arc::clone(&lost);
1412 let task_lock_loss = lock_loss.clone();
1413 tokio::spawn(async move {
1414 tokio::select! {
1415 lost = futures::StreamExt::next(&mut waits) => {
1416 if lost.is_some() {
1417 task_lost.store(true, Ordering::Release);
1418 task_lock_loss.send_replace(true);
1419 }
1420 }
1421 _ = shutdown_rx.changed() => {}
1422 }
1423 });
1424 }
1425 Self {
1426 guards,
1427 lost,
1428 lock_loss,
1429 _monitor_shutdown: monitor_shutdown,
1430 }
1431 }
1432
1433 fn check(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1434 if self.is_lost() || self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
1435 self.lost.store(true, Ordering::Release);
1436 self.lock_loss.send_replace(true);
1437 return Err(SnapshotConsistencyError::LockLost);
1438 }
1439 Ok(())
1440 }
1441
1442 fn is_lost(&self) -> bool {
1443 self.lost.load(Ordering::Acquire)
1444 }
1445
1446 fn subscribe_lock_loss(&self) -> tokio::sync::watch::Receiver<bool> {
1447 self.lock_loss.subscribe()
1448 }
1449}
1450
1451pub struct SelectObjectSnapshot {
1453 pool: Arc<Sets>,
1454 bucket: String,
1455 object: String,
1456 headers: HeaderMap,
1457 opts: ObjectOptions,
1458 object_info: ObjectInfo,
1459 logical_size: u64,
1460 read_semantics_identity: [u8; 32],
1461 first_metadata: ParkingMutex<Option<crate::set_disk::PreparedGetObjectMetadata>>,
1462 lease: Arc<SelectObjectSnapshotLease>,
1463}
1464
1465impl fmt::Debug for SelectObjectSnapshot {
1466 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1467 formatter
1468 .debug_struct("SelectObjectSnapshot")
1469 .field("bucket", &self.bucket)
1470 .field("object", &self.object)
1471 .field("logical_size", &self.logical_size)
1472 .finish_non_exhaustive()
1473 }
1474}
1475
1476impl SelectObjectSnapshot {
1477 pub fn is_for(&self, bucket: &str, object: &str) -> bool {
1478 self.bucket == bucket && self.object == encode_dir_object(object)
1479 }
1480
1481 pub fn object_info(&self) -> &ObjectInfo {
1482 &self.object_info
1483 }
1484
1485 pub fn logical_size(&self) -> u64 {
1486 self.logical_size
1487 }
1488
1489 pub fn matches_version(&self, requested: &str) -> bool {
1490 select_snapshot_version_matches(self.object_info.version_id, requested)
1491 }
1492
1493 pub fn ensure_valid(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1494 self.lease.check()
1495 }
1496
1497 #[cfg(feature = "test-util")]
1498 pub fn mark_lost_for_test(&self) {
1499 self.lease.lost.store(true, Ordering::Release);
1500 self.lease.lock_loss.send_replace(true);
1501 }
1502
1503 pub async fn open_reader(
1504 &self,
1505 range: Option<HTTPRangeSpec>,
1506 ) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
1507 self.lease.check()?;
1508 let first_metadata = self.first_metadata.lock().take();
1509 let metadata = match first_metadata {
1510 Some(metadata) => metadata,
1511 None => {
1512 self.pool
1513 .prepare_get_object_reader_metadata(&self.bucket, &self.object, &self.opts)
1514 .await?
1515 }
1516 };
1517 self.lease.check()?;
1518 if metadata.read_semantics_identity() != self.read_semantics_identity {
1519 return Err(SnapshotConsistencyError::ObjectChanged.into());
1520 }
1521
1522 let mut reader =
1523 crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
1524 &self.bucket,
1525 &self.object,
1526 range,
1527 self.headers.clone(),
1528 &self.opts,
1529 metadata,
1530 ))
1531 .await?;
1532 self.lease.check()?;
1533 reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
1534 reader.stream = Box::new(SelectObjectSnapshotReader {
1535 inner: reader.stream,
1536 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(self.lease.subscribe_lock_loss()),
1537 lease: Arc::clone(&self.lease),
1538 });
1539 Ok(reader)
1540 }
1541}
1542
1543fn select_snapshot_version_matches(actual: Option<Uuid>, requested: &str) -> bool {
1544 let requested = requested.trim();
1545 let requested = if requested.eq_ignore_ascii_case("null") {
1546 Uuid::nil()
1547 } else if let Ok(requested) = Uuid::parse_str(requested) {
1548 requested
1549 } else {
1550 return false;
1551 };
1552 actual.unwrap_or_else(Uuid::nil) == requested
1553}
1554
1555struct SelectObjectSnapshotReader {
1556 inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
1557 lock_loss_wake: SelectObjectSnapshotLockLossWake,
1558 lease: Arc<SelectObjectSnapshotLease>,
1559}
1560
1561struct SelectObjectSnapshotLockLossWake {
1562 stream: tokio_stream::wrappers::WatchStream<bool>,
1563}
1564
1565impl SelectObjectSnapshotLockLossWake {
1566 fn new(receiver: tokio::sync::watch::Receiver<bool>) -> Self {
1567 Self {
1568 stream: tokio_stream::wrappers::WatchStream::new(receiver),
1569 }
1570 }
1571
1572 fn poll_lost(&mut self, cx: &mut Context<'_>) -> bool {
1573 loop {
1574 match futures::Stream::poll_next(Pin::new(&mut self.stream), cx) {
1575 Poll::Ready(Some(true)) => return true,
1576 Poll::Ready(Some(false)) => {}
1577 Poll::Ready(None) | Poll::Pending => return false,
1578 }
1579 }
1580 }
1581}
1582
1583impl AsyncRead for SelectObjectSnapshotReader {
1584 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
1585 if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() {
1586 return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1587 }
1588 let filled_before = buf.filled().len();
1589 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1590 if self.lease.check().is_err() {
1591 buf.set_filled(filled_before);
1592 return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1593 }
1594 poll
1595 }
1596}
1597
1598fn log_object_lock_acquire_if_slow(
1599 op: &'static str,
1600 bucket: &str,
1601 object: &str,
1602 owner: Option<&str>,
1603 mode: ObjectLockDiagMode,
1604 elapsed: Duration,
1605 diag_enabled: bool,
1606) {
1607 if !diag_enabled {
1608 return;
1609 }
1610
1611 let threshold = get_object_lock_diag_slow_acquire_threshold();
1612 record_object_lock_diag_acquire_duration(op, mode.as_str(), elapsed);
1613 if elapsed >= threshold {
1614 record_object_lock_diag_slow_acquire(op, mode.as_str());
1615 warn!(
1616 target: "rustfs_ecstore::object_lock_diag",
1617 op,
1618 bucket,
1619 object,
1620 mode = %mode,
1621 owner = owner.unwrap_or_default(),
1622 acquire_ms = elapsed.as_millis(),
1623 threshold_ms = threshold.as_millis(),
1624 "object namespace lock acquisition exceeded threshold"
1625 );
1626 }
1627}
1628
1629fn select_data_movement_target_pool(
1630 existing_pool_idx: Result<usize>,
1631 src_pool_idx: usize,
1632 delete_marker: bool,
1633) -> Result<Option<usize>> {
1634 match existing_pool_idx {
1635 Ok(pool_idx) => {
1636 if delete_marker && pool_idx == src_pool_idx {
1637 Ok(None)
1638 } else {
1639 Ok(Some(pool_idx))
1640 }
1641 }
1642 Err(err) => {
1643 if is_err_read_quorum(&err) {
1644 return Err(StorageError::ErasureWriteQuorum);
1645 }
1646 if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
1647 Ok(None)
1648 } else {
1649 Err(err)
1650 }
1651 }
1652 }
1653}
1654
1655fn latest_object_access_delete_marker_error(
1656 bucket: &str,
1657 object: &str,
1658 info: &ObjectInfo,
1659 opts: &ObjectOptions,
1660) -> Option<Error> {
1661 if !info.delete_marker {
1662 return None;
1663 }
1664
1665 Some(if opts.version_id.is_none() || opts.delete_marker {
1666 to_object_err(StorageError::FileNotFound, vec![bucket, object])
1667 } else {
1668 to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])
1669 })
1670}
1671
1672fn resolve_latest_object_access(
1673 bucket: &str,
1674 object: &str,
1675 info: ObjectInfo,
1676 idx: usize,
1677 opts: &ObjectOptions,
1678) -> Result<(ObjectInfo, usize)> {
1679 if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) {
1680 return Err(err);
1681 }
1682
1683 Ok((info, idx))
1684}
1685
1686fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
1687 (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
1688}
1689
1690fn latest_versioned_delete_creates_distinct_marker(opts: &ObjectOptions) -> bool {
1691 opts.versioned
1692 && !opts.version_suspended
1693 && opts.version_id.is_none()
1694 && !opts.delete_marker
1695 && !opts.data_movement
1696 && !opts.replication_request
1697}
1698
1699#[cfg(any(test, feature = "test-util"))]
1700struct DeleteAfterObjectLockSnapshotBarrierState {
1701 bucket: String,
1702 arrived: tokio::sync::Notify,
1703 release: tokio::sync::Notify,
1704 namespace_pending: tokio::sync::Notify,
1705 namespace_acquired: AtomicBool,
1706}
1707
1708#[cfg(any(test, feature = "test-util"))]
1709pub struct DeleteAfterObjectLockSnapshotBarrier {
1710 state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
1711}
1712
1713#[cfg(any(test, feature = "test-util"))]
1714static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
1715 std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
1716> = std::sync::OnceLock::new();
1717
1718#[cfg(any(test, feature = "test-util"))]
1719impl DeleteAfterObjectLockSnapshotBarrier {
1720 pub fn install(bucket: &str) -> Self {
1721 let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
1722 bucket: bucket.to_string(),
1723 arrived: tokio::sync::Notify::new(),
1724 release: tokio::sync::Notify::new(),
1725 namespace_pending: tokio::sync::Notify::new(),
1726 namespace_acquired: AtomicBool::new(false),
1727 });
1728 let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1729 .get_or_init(|| std::sync::Mutex::new(None))
1730 .lock()
1731 .expect("delete snapshot barrier mutex should not poison");
1732 assert!(slot.is_none(), "delete snapshot barrier must not already be installed");
1733 *slot = Some(Arc::clone(&state));
1734 Self { state }
1735 }
1736
1737 pub async fn wait_until_paused(&self) {
1738 self.state.arrived.notified().await;
1739 }
1740
1741 pub fn release(&self) {
1742 self.state.release.notify_one();
1743 }
1744
1745 pub async fn release_and_wait_until_namespace_pending(&self) {
1746 let namespace_pending = self.state.namespace_pending.notified();
1747 self.release();
1748 tokio::time::timeout(Duration::from_secs(5), namespace_pending)
1749 .await
1750 .expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
1751 }
1752
1753 pub fn namespace_acquired(&self) -> bool {
1754 self.state.namespace_acquired.load(Ordering::Acquire)
1755 }
1756}
1757
1758#[cfg(any(test, feature = "test-util"))]
1759impl Drop for DeleteAfterObjectLockSnapshotBarrier {
1760 fn drop(&mut self) {
1761 self.state.release.notify_one();
1762 if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() {
1763 let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison");
1764 if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
1765 *slot = None;
1766 }
1767 }
1768 }
1769}
1770
1771#[cfg(any(test, feature = "test-util"))]
1772async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
1773 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1774 .get_or_init(|| std::sync::Mutex::new(None))
1775 .lock()
1776 .expect("delete snapshot barrier mutex should not poison")
1777 .as_ref()
1778 .filter(|state| state.bucket == bucket)
1779 .cloned();
1780 if let Some(state) = state {
1781 state.arrived.notify_one();
1782 state.release.notified().await;
1783 }
1784}
1785
1786#[cfg(any(test, feature = "test-util"))]
1787fn notify_delete_namespace_pending(bucket: &str) {
1788 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1789 .get_or_init(|| std::sync::Mutex::new(None))
1790 .lock()
1791 .expect("delete snapshot barrier mutex should not poison")
1792 .as_ref()
1793 .filter(|state| state.bucket == bucket)
1794 .cloned();
1795 if let Some(state) = state {
1796 state.namespace_pending.notify_one();
1797 }
1798}
1799
1800#[cfg(any(test, feature = "test-util"))]
1801fn notify_delete_namespace_acquired(bucket: &str) {
1802 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1803 .get_or_init(|| std::sync::Mutex::new(None))
1804 .lock()
1805 .expect("delete snapshot barrier mutex should not poison")
1806 .as_ref()
1807 .filter(|state| state.bucket == bucket)
1808 .cloned();
1809 if let Some(state) = state {
1810 state.namespace_acquired.store(true, Ordering::Release);
1811 }
1812}
1813
1814#[cfg(test)]
1815struct VersionedDeleteMarkerCommitBarrierState {
1816 bucket: String,
1817 object: String,
1818 arrived: tokio::sync::Notify,
1819 release: tokio::sync::Notify,
1820}
1821
1822#[cfg(test)]
1823pub(crate) struct VersionedDeleteMarkerCommitBarrier {
1824 state: Arc<VersionedDeleteMarkerCommitBarrierState>,
1825}
1826
1827#[cfg(test)]
1828static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock<
1829 std::sync::Mutex<Option<Arc<VersionedDeleteMarkerCommitBarrierState>>>,
1830> = std::sync::OnceLock::new();
1831
1832#[cfg(test)]
1833impl VersionedDeleteMarkerCommitBarrier {
1834 pub(crate) fn install(bucket: &str, object: &str) -> Self {
1835 let state = Arc::new(VersionedDeleteMarkerCommitBarrierState {
1836 bucket: bucket.to_string(),
1837 object: object.to_string(),
1838 arrived: tokio::sync::Notify::new(),
1839 release: tokio::sync::Notify::new(),
1840 });
1841 let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1842 .get_or_init(|| std::sync::Mutex::new(None))
1843 .lock()
1844 .expect("versioned delete-marker commit barrier mutex should not poison");
1845 assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique");
1846 *slot = Some(Arc::clone(&state));
1847 Self { state }
1848 }
1849
1850 pub(crate) async fn wait_until_paused(&self) {
1851 tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1852 .await
1853 .expect("versioned DELETE should reach the post-marker-commit barrier");
1854 }
1855
1856 pub(crate) fn release(&self) {
1857 self.state.release.notify_one();
1858 }
1859}
1860
1861#[cfg(test)]
1862impl Drop for VersionedDeleteMarkerCommitBarrier {
1863 fn drop(&mut self) {
1864 self.state.release.notify_one();
1865 let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1866 .get_or_init(|| std::sync::Mutex::new(None))
1867 .lock()
1868 .expect("versioned delete-marker commit barrier mutex should not poison");
1869 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1870 *slot = None;
1871 }
1872 }
1873}
1874
1875#[cfg(test)]
1876async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) {
1877 let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1878 .get_or_init(|| std::sync::Mutex::new(None))
1879 .lock()
1880 .expect("versioned delete-marker commit barrier mutex should not poison")
1881 .as_ref()
1882 .filter(|state| state.bucket == bucket && state.object == object)
1883 .cloned();
1884 if let Some(state) = state {
1885 state.arrived.notify_one();
1886 state.release.notified().await;
1887 }
1888}
1889
1890fn should_purge_orphan_dir_on_missing(err: &Error, object: &str) -> bool {
1899 (is_err_object_not_found(err) || is_err_version_not_found(err)) && rustfs_utils::path::is_dir_object(object)
1900}
1901
1902fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1903 let mut lookup_opts = opts.clone();
1904 lookup_opts.no_lock = no_lock;
1905 if lookup_opts.version_id.is_some() {
1906 lookup_opts.metadata_chg = true;
1907 }
1908
1909 lookup_opts
1910}
1911
1912fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1913 writer_pool_lookup_opts(opts, no_lock)
1914}
1915
1916fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool {
1917 opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some())
1918}
1919
1920fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1921 let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
1922 lookup_opts.skip_decommissioned = true;
1923 lookup_opts.skip_rebalancing = true;
1924
1925 lookup_opts
1926}
1927
1928fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1929 let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock);
1930 lookup_opts.skip_decommissioned = opts.data_movement;
1931 lookup_opts
1932}
1933
1934fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
1935 pool_count > 0 && delete_only_releases_capacity(opts)
1936}
1937
1938fn delete_only_releases_capacity(opts: &ObjectOptions) -> bool {
1939 !opts.versioned && !opts.version_suspended || opts.version_id.is_some()
1940}
1941
1942fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
1943 if object.version_id.is_some() {
1944 return false;
1945 }
1946
1947 let object_name = decode_dir_object(&object.object_name);
1948 let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name);
1949 versioned || version_suspended
1950}
1951
1952fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option<usize>, pool_idx: usize) -> bool {
1953 !creates_latest_marker || marker_target_pool_idx == Some(pool_idx)
1954}
1955
1956#[cfg(test)]
1957struct BatchDeletePoolErrorInjectionState {
1958 bucket: String,
1959 pool_idx: usize,
1960 errors: std::collections::HashMap<String, Error>,
1961 observed: std::sync::atomic::AtomicUsize,
1962}
1963
1964#[cfg(test)]
1965pub(crate) struct BatchDeletePoolErrorInjection {
1966 state: Arc<BatchDeletePoolErrorInjectionState>,
1967}
1968
1969#[cfg(test)]
1970static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock<std::sync::Mutex<Option<Arc<BatchDeletePoolErrorInjectionState>>>> =
1971 std::sync::OnceLock::new();
1972
1973#[cfg(test)]
1974impl BatchDeletePoolErrorInjection {
1975 pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self {
1976 let state = Arc::new(BatchDeletePoolErrorInjectionState {
1977 bucket: bucket.to_string(),
1978 pool_idx,
1979 errors: errors.into_iter().collect(),
1980 observed: std::sync::atomic::AtomicUsize::new(0),
1981 });
1982 let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
1983 .get_or_init(|| std::sync::Mutex::new(None))
1984 .lock()
1985 .expect("batch delete pool error injection mutex should not poison");
1986 assert!(slot.is_none(), "batch delete pool error injection must be unique");
1987 *slot = Some(Arc::clone(&state));
1988 Self { state }
1989 }
1990
1991 pub(crate) fn observed(&self) -> usize {
1992 self.state.observed.load(Ordering::Acquire)
1993 }
1994}
1995
1996#[cfg(test)]
1997impl Drop for BatchDeletePoolErrorInjection {
1998 fn drop(&mut self) {
1999 let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
2000 .get_or_init(|| std::sync::Mutex::new(None))
2001 .lock()
2002 .expect("batch delete pool error injection mutex should not poison");
2003 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
2004 *slot = None;
2005 }
2006 }
2007}
2008
2009#[cfg(test)]
2010fn inject_batch_delete_pool_errors(
2011 bucket: &str,
2012 pool_idx: usize,
2013 object_names: &[String],
2014 deleted: &[DeletedObject],
2015 errors: &mut [Option<Error>],
2016) {
2017 let state = BATCH_DELETE_POOL_ERROR_INJECTION
2018 .get_or_init(|| std::sync::Mutex::new(None))
2019 .lock()
2020 .expect("batch delete pool error injection mutex should not poison")
2021 .as_ref()
2022 .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx)
2023 .cloned();
2024 let Some(state) = state else {
2025 return;
2026 };
2027
2028 for (idx, object_name) in object_names.iter().enumerate() {
2029 let Some(error) = state.errors.get(object_name) else {
2030 continue;
2031 };
2032 if errors[idx].is_none() && deleted[idx].found {
2033 errors[idx] = Some(error.clone());
2034 state.observed.fetch_add(1, Ordering::AcqRel);
2035 }
2036 }
2037}
2038
2039fn resolve_batch_delete_pool_results<'a>(
2040 initial_error: Option<Error>,
2041 pool_results: impl IntoIterator<Item = (&'a DeletedObject, &'a Option<Error>)>,
2042) -> (Option<DeletedObject>, Option<Error>, bool) {
2043 let mut failure = initial_error.map(|err| (None, err));
2044 let mut deleted = None;
2045 let mut fallback: Option<(DeletedObject, Option<Error>)> = None;
2046 let mut attempted = false;
2047
2048 for (pool_delete, pool_error) in pool_results {
2049 attempted = true;
2050 match pool_error {
2051 Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => {
2052 if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) {
2053 fallback = Some(((*pool_delete).clone(), Some(err.clone())));
2054 }
2055 }
2056 Some(err) => {
2057 if failure.is_none() {
2058 failure = Some((Some((*pool_delete).clone()), err.clone()));
2059 }
2060 }
2061 None if pool_delete.found => {
2062 if deleted.is_none() {
2063 deleted = Some((*pool_delete).clone());
2064 }
2065 }
2066 None => {
2067 if fallback.is_none() {
2068 fallback = Some(((*pool_delete).clone(), None));
2069 }
2070 }
2071 }
2072 }
2073
2074 if let Some((failed_delete, err)) = failure {
2075 return (failed_delete, Some(err), attempted);
2076 }
2077 if let Some(deleted) = deleted {
2078 return (Some(deleted), None, attempted);
2079 }
2080 if let Some((deleted, err)) = fallback {
2081 return (Some(deleted), err, attempted);
2082 }
2083
2084 (None, None, attempted)
2085}
2086
2087fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
2088 let mut lookup_opts = opts.clone();
2089 lookup_opts.no_lock = false;
2093 lookup_opts.skip_decommissioned = true;
2094 lookup_opts.skip_rebalancing = true;
2095 lookup_opts
2096}
2097
2098fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
2099 info.get_actual_size().ok()
2100}
2101
2102pub(super) fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2103 is_data_movement_delete_marker(source)
2104 && is_data_movement_delete_marker(target)
2105 && source.version_id == target.version_id
2106 && source.mod_time == target.mod_time
2107 && is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
2108 && source.user_tags == target.user_tags
2109 && source.replication_status_internal == target.replication_status_internal
2110 && source.replication_status == target.replication_status
2111 && source.version_purge_status_internal == target.version_purge_status_internal
2112 && source.version_purge_status == target.version_purge_status
2113}
2114
2115fn is_equivalent_data_movement_delete_marker_metadata(
2116 source: &HashMap<String, String>,
2117 target: &HashMap<String, String>,
2118) -> bool {
2119 matches!(
2120 (
2121 data_movement_delete_marker_metadata_identity(source),
2122 data_movement_delete_marker_metadata_identity(target)
2123 ),
2124 (Some(source), Some(target)) if source == target
2125 )
2126}
2127
2128fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
2129 let mut identity = HashMap::with_capacity(metadata.len());
2130 let mut local_tier_free_version_id = None;
2131 for (key, value) in metadata {
2132 let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
2133 identity.insert(key.clone(), value.clone());
2134 continue;
2135 };
2136
2137 if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
2138 let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
2139 if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
2140 return None;
2141 }
2142 local_tier_free_version_id = Some(version_id);
2143 continue;
2144 }
2145 if suffix.eq_ignore_ascii_case(rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID) {
2146 continue;
2147 }
2148
2149 let canonical_suffix = [
2150 rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
2151 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
2152 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2153 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2154 rustfs_utils::http::SUFFIX_PURGESTATUS,
2155 ]
2156 .into_iter()
2157 .find(|candidate| suffix.eq_ignore_ascii_case(candidate))
2158 .map(str::to_string)
2159 .or_else(|| {
2160 [
2161 rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
2162 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
2163 ]
2164 .into_iter()
2165 .find_map(|prefix| {
2166 suffix
2167 .get(..prefix.len())
2168 .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
2169 .then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
2170 })
2171 })
2172 .unwrap_or_else(|| suffix.to_string());
2173 let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
2174 || canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
2175 {
2176 rustfs_filemeta::parse_replication_timestamp(value)?
2177 .unix_timestamp_nanos()
2178 .to_string()
2179 } else {
2180 value.clone()
2181 };
2182 let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2183 if identity
2184 .insert(canonical_key, canonical_value.clone())
2185 .is_some_and(|existing| existing != canonical_value)
2186 {
2187 return None;
2188 }
2189 }
2190 for (status_suffix, timestamp_suffix) in [
2191 (rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
2192 (
2193 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2194 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2195 ),
2196 ] {
2197 let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2198 let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2199 match (identity.contains_key(&status_key), identity.contains_key(×tamp_key)) {
2200 (true, false) => {
2201 identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
2202 }
2203 (false, true) => return None,
2204 _ => {}
2205 }
2206 }
2207 Some(identity)
2208}
2209
2210fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
2211 info.delete_marker
2212}
2213
2214fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
2215 is_data_movement_delete_marker(source)
2216 && source.mod_time.is_some()
2217 && source.mod_time == expected_mod_time
2218 && data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
2219}
2220
2221fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
2222 let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
2223 let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
2224 let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
2225 (None, None) => Default::default(),
2226 (Some(status), timestamp) => {
2227 let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2228 if status.is_empty() {
2229 return None;
2230 }
2231 let timestamp = match timestamp {
2232 Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
2233 None => OffsetDateTime::UNIX_EPOCH,
2234 };
2235 (status, Some(timestamp))
2236 }
2237 (None, Some(_)) => return None,
2238 };
2239 let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
2240 let replication_timestamp =
2241 rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
2242 let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
2243 (None, None) => Default::default(),
2244 (Some(status), timestamp) => {
2245 let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2246 let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
2247 if direct_status.is_empty() && targets.is_empty() {
2248 return None;
2249 }
2250 let timestamp = match timestamp {
2251 Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
2252 None => OffsetDateTime::UNIX_EPOCH,
2253 };
2254 (Some(status), Some(timestamp), targets)
2255 }
2256 (None, Some(_)) => return None,
2257 };
2258 let mut state = source.replication_state();
2259 if state.target_delete_marker_version_ids_corrupt {
2260 return None;
2261 }
2262 state.replica_status = replica_status;
2263 state.replica_timestamp = replica_timestamp;
2264 state.replication_status_internal = replication_status;
2265 state.replication_timestamp = replication_timestamp;
2266 state.targets = replication_targets;
2267 state.replicate_decision_str = source.replication_decision.clone();
2268 state.delete_marker = true;
2269
2270 let mut target_opts = opts.clone();
2271 target_opts.mod_time = source.mod_time;
2272 target_opts.delete_replication = Some(state);
2273 Some(target_opts)
2274}
2275
2276fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
2277 ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
2278}
2279
2280fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
2281 let expected = expected_data_movement_tiered_object(source);
2282 let Some(source_actual_size) = effective_object_actual_size(&expected) else {
2283 return false;
2284 };
2285 let Some(target_actual_size) = effective_object_actual_size(target) else {
2286 return false;
2287 };
2288
2289 source.version_id == target.version_id
2290 && !target.delete_marker
2291 && source.size == target.size
2292 && source.get_etag() == target.etag
2293 && source.checksum == target.checksum
2294 && crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
2295 && source.mod_time == target.mod_time
2296 && crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
2297 && expected.user_tags == target.user_tags
2298 && expected.expires == target.expires
2299 && expected.storage_class == target.storage_class
2300 && expected.replication_status_internal == target.replication_status_internal
2301 && expected.replication_status == target.replication_status
2302 && expected.version_purge_status_internal == target.version_purge_status_internal
2303 && expected.version_purge_status == target.version_purge_status
2304 && expected.transitioned_object.status == target.transitioned_object.status
2305 && expected.transition_version_state == target.transition_version_state
2306 && expected.transitioned_object.name == target.transitioned_object.name
2307 && expected.transitioned_object.tier == target.transitioned_object.tier
2308 && expected.transitioned_object.version_id == target.transitioned_object.version_id
2309 && expected.transitioned_object.free_version == target.transitioned_object.free_version
2310 && source_actual_size == target_actual_size
2311}
2312
2313pub(crate) fn tiered_data_movement_source_matches(
2314 expected: &rustfs_filemeta::FileInfo,
2315 current: &rustfs_filemeta::FileInfo,
2316) -> Result<bool> {
2317 let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
2318 let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
2319 Ok(expected.version_id == current.version_id
2320 && expected.deleted == current.deleted
2321 && expected.tier_free_version() == current.tier_free_version()
2322 && expected.data_dir == current.data_dir
2323 && expected.mod_time == current.mod_time
2324 && expected.size == current.size
2325 && expected.get_etag() == current.get_etag()
2326 && expected.transition_status == current.transition_status
2327 && expected.transitioned_objname == current.transitioned_objname
2328 && expected.transition_tier == current.transition_tier
2329 && expected.transition_version_id == current.transition_version_id
2330 && expected.transition_version == current.transition_version
2331 && expected.transition_version_state == current.transition_version_state
2332 && expected_backend == current_backend)
2333}
2334
2335fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
2336 StorageError::DataMovementOverwriteErr(
2337 bucket.to_owned(),
2338 object.to_owned(),
2339 version_id.map(|id| id.to_string()).unwrap_or_default(),
2340 )
2341}
2342
2343fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
2344 target_pool_idx != src_pool_idx
2345}
2346
2347fn resolve_data_movement_resume_target_pool(
2348 selected_target_pool_idx: usize,
2349 resume_target_pool_idx: Option<usize>,
2350 src_pool_idx: usize,
2351) -> usize {
2352 if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
2353 selected_target_pool_idx
2354 } else {
2355 resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
2356 }
2357}
2358
2359fn resolve_data_movement_delete_marker_resume_result(
2360 target_result: Result<Option<ObjectInfo>>,
2361 source: &ObjectInfo,
2362 src_pool_idx: usize,
2363 target_pool_idx: usize,
2364) -> Result<bool> {
2365 if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2366 return Ok(false);
2367 }
2368
2369 let Some(target) = target_result? else {
2370 return Ok(false);
2371 };
2372
2373 Ok(is_equivalent_data_movement_delete_marker(source, &target))
2374}
2375
2376fn resolve_data_movement_tiered_resume_result(
2377 target_result: Result<Option<ObjectInfo>>,
2378 source: &rustfs_filemeta::FileInfo,
2379 src_pool_idx: usize,
2380 target_pool_idx: usize,
2381) -> Result<bool> {
2382 if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2383 return Ok(false);
2384 }
2385
2386 let Some(target) = target_result? else {
2387 return Ok(false);
2388 };
2389
2390 Ok(is_equivalent_data_movement_tiered_object(source, &target))
2391}
2392
2393fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
2394 let del_objects = objects
2395 .iter()
2396 .map(|object| DeletedObject {
2397 object_name: decode_dir_object(&object.object_name),
2398 version_id: object.version_id,
2399 ..Default::default()
2400 })
2401 .collect();
2402 let del_errs = objects.iter().map(|_| Some(err.clone())).collect();
2403
2404 (del_objects, del_errs)
2405}
2406
2407fn return_batch_delete_lock_error_with_accounting(
2408 objects: &[ObjectToDelete],
2409 err: Error,
2410) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
2411 let (deleted, errors) = return_batch_delete_lock_error(objects, err);
2412 (deleted, errors, vec![None; objects.len()])
2413}
2414
2415fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
2416 let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
2417 object_names.sort_unstable();
2418 object_names.dedup();
2419 object_names
2420}
2421
2422fn materialized_copy_user_defined(source: &HashMap<String, String>) -> HashMap<String, String> {
2433 let mut metadata = source.clone();
2434 for suffix in [
2435 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2436 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2437 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2438 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2439 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2440 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2441 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2442 rustfs_utils::http::SUFFIX_FREE_VERSION,
2443 rustfs_utils::http::SUFFIX_TIER_FV_ID,
2444 rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
2445 ] {
2446 rustfs_utils::http::metadata_compat::remove_str(&mut metadata, suffix);
2447 }
2448 metadata
2449}
2450
2451enum RemoteTuplePublicationSource {
2452 Object(ObjectInfo),
2453 Tiered(rustfs_filemeta::FileInfo),
2454}
2455
2456#[must_use = "the fixed read anchor must remain live until publication or migration abort"]
2461pub(crate) struct DecommissionFixedReadAnchor {
2462 guard: ObjectLockDiagGuard,
2463}
2464
2465impl DecommissionFixedReadAnchor {
2466 pub(crate) fn guard(&self) -> &ObjectLockDiagGuard {
2467 &self.guard
2468 }
2469
2470 fn is_lock_lost(&self) -> bool {
2471 self.guard.is_lock_lost()
2472 }
2473}
2474
2475#[must_use = "a data-movement publication fence must be consumed by the final target commit"]
2484pub(crate) struct RemoteTuplePublicationFence {
2485 store: Arc<ECStore>,
2486 bucket: String,
2487 object: String,
2488 source_pool_idx: usize,
2489 source: RemoteTuplePublicationSource,
2490 include_fixed_domain: bool,
2491 fixed_read_anchor: Option<DecommissionFixedReadAnchor>,
2492 backend_target: Option<(String, Option<TierDestinationId>)>,
2493}
2494
2495#[must_use = "the publication commit guard must live through the target metadata commit"]
2498pub(crate) struct RemoteTuplePublicationCommitGuard {
2499 guards: Vec<ObjectLockDiagGuard>,
2500 backend_lease: Option<TierOperationLease>,
2501 revalidated_tiered_source: Option<rustfs_filemeta::FileInfo>,
2502}
2503
2504impl RemoteTuplePublicationCommitGuard {
2505 pub(crate) fn is_lock_lost(&self) -> bool {
2506 self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2507 || self
2508 .backend_lease
2509 .as_ref()
2510 .is_some_and(|lease| !lease.is_current_generation())
2511 }
2512
2513 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
2514 for guard in &self.guards {
2515 guard.add_namespace_lock_fence(opts);
2516 }
2517 }
2518
2519 pub(crate) fn revalidated_tiered_source(&self) -> Option<&rustfs_filemeta::FileInfo> {
2520 self.revalidated_tiered_source.as_ref()
2521 }
2522}
2523
2524fn remote_tuple_publication_target(source: &ObjectInfo) -> std::io::Result<Option<(String, Option<TierDestinationId>)>> {
2525 if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !source.transitioned_object.free_version {
2526 return Ok(None);
2527 }
2528 if source.transitioned_object.tier.is_empty() || source.transitioned_object.name.is_empty() {
2529 return Err(std::io::Error::other(
2530 "remote tuple publication source is missing its tier or remote object identity",
2531 ));
2532 }
2533 let backend_identity = tier_destination_id_from_metadata(source.user_defined.as_ref())?;
2534 Ok(Some((source.transitioned_object.tier.clone(), backend_identity)))
2535}
2536
2537fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current: &ObjectInfo) -> bool {
2538 let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else {
2539 return false;
2540 };
2541
2542 expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil())
2543 && expected.data_dir == current.data_dir
2544 && expected.version_id == current.version_id
2545 && expected.delete_marker == current.delete_marker
2546 && expected.size == current.size
2547 && expected_actual_size == current_actual_size
2548 && expected.etag == current.etag
2549 && expected.checksum == current.checksum
2550 && expected.mod_time == current.mod_time
2551 && expected.storage_class == current.storage_class
2552 && crate::data_movement::is_equivalent_data_movement_metadata(
2553 expected,
2554 current,
2555 expected_actual_size,
2556 current_actual_size,
2557 )
2558 && expected.user_tags == current.user_tags
2559 && expected.expires == current.expires
2560 && expected.replication_status_internal == current.replication_status_internal
2561 && expected.replication_status == current.replication_status
2562 && expected.version_purge_status_internal == current.version_purge_status_internal
2563 && expected.version_purge_status == current.version_purge_status
2564 && expected.transitioned_object.name == current.transitioned_object.name
2565 && expected.transitioned_object.version_id == current.transitioned_object.version_id
2566 && expected.transitioned_object.tier == current.transitioned_object.tier
2567 && expected.transitioned_object.free_version == current.transitioned_object.free_version
2568 && expected.transitioned_object.status == current.transitioned_object.status
2569 && expected.transition_version_state == current.transition_version_state
2570 && remote_tuple_publication_parts_match(&expected.parts, ¤t.parts)
2571}
2572
2573fn remote_tuple_publication_parts_match(expected: &[ObjectPartInfo], current: &[ObjectPartInfo]) -> bool {
2574 if expected.len() != current.len() {
2575 return false;
2576 }
2577 let Some(mut current_parts) = crate::data_movement::data_movement_parts_by_number(current) else {
2578 return false;
2579 };
2580
2581 expected.iter().all(|expected_part| {
2582 current_parts.remove(&expected_part.number).is_some_and(|current_part| {
2583 current_part.size == expected_part.size
2584 && current_part.actual_size == expected_part.actual_size
2585 && current_part.etag == expected_part.etag
2586 })
2587 })
2588}
2589
2590impl RemoteTuplePublicationFence {
2591 pub(crate) fn under_fixed_read_anchor(mut self, anchor: DecommissionFixedReadAnchor) -> Result<Self> {
2592 if self.include_fixed_domain {
2593 return Err(Error::other(
2594 "data movement publication capability cannot attach a fixed read anchor after requesting fixed write",
2595 ));
2596 }
2597 if anchor.is_lock_lost() {
2598 return Err(Error::other(
2599 "data movement publication capability received an already-lost fixed read anchor",
2600 ));
2601 }
2602 self.include_fixed_domain = self.backend_target.is_some();
2603 self.fixed_read_anchor = Some(anchor);
2604 Ok(self)
2605 }
2606
2607 pub(crate) fn fixed_read_anchor_guard(&self) -> Option<&ObjectLockDiagGuard> {
2608 self.fixed_read_anchor.as_ref().map(DecommissionFixedReadAnchor::guard)
2609 }
2610
2611 pub(crate) async fn into_commit_guard(
2612 self,
2613 target_pool_idx: usize,
2614 bucket: &str,
2615 object: &str,
2616 ) -> Result<RemoteTuplePublicationCommitGuard> {
2617 let Self {
2618 store,
2619 bucket: expected_bucket,
2620 object: expected_object,
2621 source_pool_idx,
2622 source,
2623 include_fixed_domain,
2624 fixed_read_anchor,
2625 backend_target,
2626 } = self;
2627 if bucket != expected_bucket || object != expected_object {
2628 return Err(Error::other_with_context(
2629 "data movement publication capability target mismatch",
2630 format!("expected {expected_bucket}/{expected_object}, got {bucket}/{object}"),
2631 ));
2632 }
2633 if fixed_read_anchor
2634 .as_ref()
2635 .is_some_and(DecommissionFixedReadAnchor::is_lock_lost)
2636 {
2637 return Err(Error::other("data movement publication fixed read anchor was lost before commit"));
2638 }
2639 drop(fixed_read_anchor);
2643 let backend_lease = if let Some((tier_name, backend_identity)) = backend_target {
2646 let manager = store.tier_config_mgr();
2647 Some(
2648 match backend_identity {
2649 Some(identity) => {
2650 TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, &tier_name, identity).await
2651 }
2652 None => TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await,
2653 }
2654 .map_err(Error::other)?,
2655 )
2656 } else {
2657 None
2658 };
2659
2660 let guards = store
2661 .acquire_data_movement_publication_write_locks(bucket, object, source_pool_idx, target_pool_idx, include_fixed_domain)
2662 .await?;
2663 if guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2664 || backend_lease.as_ref().is_some_and(|lease| !lease.is_current_generation())
2665 {
2666 return Err(Error::other("data movement publication fence was lost before source revalidation"));
2667 }
2668
2669 let source_pool = store.pools.get(source_pool_idx).ok_or_else(|| {
2670 Error::other_with_context("invalid data movement source pool", format!("pool index {source_pool_idx}"))
2671 })?;
2672 let mut revalidated_tiered_source = None;
2673 let source_matches = match source {
2674 RemoteTuplePublicationSource::Object(expected) => {
2675 let lookup_opts = ObjectOptions {
2676 versioned: expected.version_id.is_some(),
2677 version_id: expected.version_id.map(|version_id| version_id.to_string()),
2678 incl_free_versions: expected.transitioned_object.free_version,
2679 include_part_checksums: true,
2680 no_lock: true,
2681 ..Default::default()
2682 };
2683 match source_pool.get_object_info(bucket, object, &lookup_opts).await {
2684 Ok(current) => remote_tuple_publication_object_source_matches(&expected, ¤t),
2685 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => false,
2686 Err(err) => return Err(err),
2687 }
2688 }
2689 RemoteTuplePublicationSource::Tiered(expected) => {
2690 let logical_object = decode_dir_object(object);
2691 let current_versions = source_pool
2692 .get_disks_by_key(object)
2693 .load_file_info_versions_exact(bucket, &logical_object)
2694 .await?;
2695 let is_free_version = expected.tier_free_version();
2696 let Some(current) = current_versions.as_ref().and_then(|versions| {
2697 versions.versions.iter().find(|current| {
2698 current.version_id == expected.version_id && current.tier_free_version() == is_free_version
2699 })
2700 }) else {
2701 return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
2702 };
2703 if tiered_data_movement_source_matches(&expected, current)? {
2704 revalidated_tiered_source = Some(current.clone());
2705 true
2706 } else {
2707 false
2708 }
2709 }
2710 };
2711 if !source_matches {
2712 return Err(StorageError::DataMovementOverwriteErr(
2713 bucket.to_string(),
2714 object.to_string(),
2715 String::new(),
2716 ));
2717 }
2718
2719 let commit_guard = RemoteTuplePublicationCommitGuard {
2720 guards,
2721 backend_lease,
2722 revalidated_tiered_source,
2723 };
2724 if commit_guard.is_lock_lost() {
2725 return Err(Error::other("data movement publication fence was lost after source revalidation"));
2726 }
2727 Ok(commit_guard)
2728 }
2729}
2730
2731impl ECStore {
2732 pub(crate) async fn acquire_remote_tuple_publication_fence(
2737 self: &Arc<Self>,
2738 bucket: &str,
2739 source_pool_idx: usize,
2740 source: &ObjectInfo,
2741 include_fixed_domain: bool,
2742 ) -> Result<RemoteTuplePublicationFence> {
2743 if self.pools.get(source_pool_idx).is_none() {
2744 return Err(Error::other_with_context(
2745 "data movement source pool is out of range",
2746 format!("pool index {source_pool_idx} for {bucket}/{}", source.name),
2747 ));
2748 }
2749 let backend_target = remote_tuple_publication_target(source).map_err(Error::other)?;
2750 let include_fixed_domain = include_fixed_domain && backend_target.is_some();
2751 Ok(RemoteTuplePublicationFence {
2752 store: Arc::clone(self),
2753 bucket: bucket.to_string(),
2754 object: encode_dir_object(&source.name),
2755 source_pool_idx,
2756 source: RemoteTuplePublicationSource::Object(source.clone()),
2757 include_fixed_domain,
2758 fixed_read_anchor: None,
2759 backend_target,
2760 })
2761 }
2762
2763 async fn acquire_tiered_remote_tuple_publication_fence(
2764 self: &Arc<Self>,
2765 bucket: &str,
2766 object: &str,
2767 source_pool_idx: usize,
2768 source: &rustfs_filemeta::FileInfo,
2769 ) -> Result<RemoteTuplePublicationFence> {
2770 let object_info = ObjectInfo::from_file_info(source, bucket, object, true);
2771 let Some((tier_name, backend_identity)) = remote_tuple_publication_target(&object_info).map_err(Error::other)? else {
2772 return Err(Error::other("tiered data movement source has no publishable remote tuple"));
2773 };
2774 Ok(RemoteTuplePublicationFence {
2775 store: Arc::clone(self),
2776 bucket: bucket.to_string(),
2777 object: encode_dir_object(object),
2778 source_pool_idx,
2779 source: RemoteTuplePublicationSource::Tiered(source.clone()),
2780 include_fixed_domain: true,
2781 fixed_read_anchor: None,
2782 backend_target: Some((tier_name, backend_identity)),
2783 })
2784 }
2785
2786 pub(crate) fn is_equivalent_decommission_capacity_target(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2787 source.bucket == target.bucket
2788 && source.name == decode_dir_object(&target.name)
2789 && if source.delete_marker {
2790 is_equivalent_data_movement_delete_marker(source, target)
2791 } else {
2792 crate::data_movement::is_equivalent_data_movement_object_identity(source, target, true, false)
2793 }
2794 }
2795
2796 pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
2800 check_valid_bucket_name(bucket)?;
2801 let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
2802 let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2803 let (state, bucket_incarnation_id, config_revision) =
2804 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2805 if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
2806 return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
2807 }
2808 Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket(
2809 self.id,
2810 bucket,
2811 bucket_incarnation_id,
2812 config_revision,
2813 state,
2814 lifecycle_guard,
2815 metadata_guard,
2816 )))
2817 }
2818
2819 pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
2820 &self,
2821 bucket: &str,
2822 lifecycle_fence: &NamespaceLockFence,
2823 ) -> Result<Arc<ObjectLockConfigSnapshot>> {
2824 if lifecycle_fence.is_lock_lost() {
2825 return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
2826 }
2827 let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2828 let (state, bucket_incarnation_id, config_revision) =
2829 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2830 if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
2831 return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
2832 }
2833 Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
2834 self.id,
2835 bucket,
2836 bucket_incarnation_id,
2837 config_revision,
2838 state,
2839 lifecycle_fence.clone(),
2840 metadata_guard,
2841 )))
2842 }
2843
2844 pub async fn prepare_select_object_snapshot(
2850 &self,
2851 bucket: &str,
2852 object: &str,
2853 headers: &HeaderMap,
2854 opts: &ObjectOptions,
2855 ) -> std::result::Result<SelectObjectSnapshot, PrepareSelectObjectSnapshotError> {
2856 check_get_obj_args(bucket, object)?;
2857
2858 let object = encode_dir_object(object);
2859 let mut opts = opts.clone();
2860 opts.no_lock = false;
2861 opts.metadata_cache_safe = false;
2862 let read_lock_guards = self
2863 .acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
2864 .await?;
2865 if self.ctx.lock_manager().is_disabled() {
2866 return Err(SnapshotConsistencyError::LockingDisabled.into());
2867 }
2868 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2869 return Err(SnapshotConsistencyError::LockLost.into());
2870 }
2871
2872 let (mut metadata, pool) = if self.single_pool() {
2873 let pool = Arc::clone(&self.pools[0]);
2874 let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2875 (metadata, pool)
2876 } else {
2877 Box::pin(async {
2880 let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2881 let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
2882 StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
2883 })?;
2884 Ok::<_, StorageError>((metadata, pool))
2885 })
2886 .await?
2887 };
2888 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2889 return Err(SnapshotConsistencyError::LockLost.into());
2890 }
2891 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2892 return Err(error.into());
2893 }
2894
2895 let logical_size_i64 = metadata.object_info().get_actual_size().map_err(StorageError::from)?;
2896 let logical_size = u64::try_from(logical_size_i64)
2897 .map_err(|_| PrepareSelectObjectSnapshotError::InvalidLogicalSize { size: logical_size_i64 })?;
2898 let read_semantics_identity = metadata.read_semantics_identity();
2899 let object_info = metadata.take_object_info();
2900 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2901 return Err(SnapshotConsistencyError::LockLost.into());
2902 }
2903
2904 Ok(SelectObjectSnapshot {
2905 pool,
2906 bucket: bucket.to_owned(),
2907 object,
2908 headers: rustfs_utils::http::project_ssec_transport_headers(headers),
2909 opts,
2910 object_info,
2911 logical_size,
2912 read_semantics_identity,
2913 first_metadata: ParkingMutex::new(Some(metadata)),
2914 lease: Arc::new(SelectObjectSnapshotLease::new(read_lock_guards)),
2915 })
2916 }
2917
2918 pub async fn prepare_get_object_reader(
2919 &self,
2920 bucket: &str,
2921 object: &str,
2922 range: Option<HTTPRangeSpec>,
2923 headers: HeaderMap,
2924 opts: &ObjectOptions,
2925 ) -> Result<PreparedGetObjectReader> {
2926 check_get_obj_args(bucket, object)?;
2927
2928 let object = encode_dir_object(object);
2929 let mut opts = opts.clone();
2930 let read_lock_guard = self
2931 .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
2932 .await?;
2933
2934 let (metadata, pool) = if self.single_pool() {
2935 let pool = Arc::clone(&self.pools[0]);
2936 let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2937 (metadata, pool)
2938 } else {
2939 Box::pin(async {
2942 let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2943 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2944 return Err(error);
2945 }
2946 let pool = self
2947 .pools
2948 .get(pool_idx)
2949 .cloned()
2950 .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
2951 Ok((metadata, pool))
2952 })
2953 .await?
2954 };
2955
2956 Ok(PreparedGetObjectReader {
2957 pool,
2958 bucket: bucket.to_owned(),
2959 object,
2960 range,
2961 headers,
2962 opts,
2963 metadata,
2964 read_lock_guard,
2965 })
2966 }
2967
2968 fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
2969 match err {
2970 rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
2971 mode,
2972 bucket: bucket.to_string(),
2973 object: object.to_string(),
2974 required,
2975 achieved,
2976 },
2977 other => StorageError::Lock(other),
2978 }
2979 }
2980
2981 pub(super) async fn acquire_object_write_lock(
2982 &self,
2983 op: &'static str,
2984 bucket: &str,
2985 object: &str,
2986 ) -> Result<ObjectLockDiagGuard> {
2987 let diag_enabled = is_object_lock_diag_enabled();
2988 let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
2989 let acquire_start = Instant::now();
2990 #[cfg(any(test, feature = "test-util"))]
2991 if matches!(op, "delete_object" | "delete_objects") {
2992 notify_delete_namespace_pending(bucket);
2993 }
2994 let timeout = get_lock_acquire_timeout();
2995 let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", timeout);
2996 let result = ns_lock.get_write_lock(timeout).await;
2997 attempt.observe(&result);
2998 let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
2999 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3000 log_object_lock_acquire_if_slow(
3001 op,
3002 bucket,
3003 object,
3004 owner.as_deref(),
3005 ObjectLockDiagMode::Write,
3006 acquire_start.elapsed(),
3007 diag_enabled,
3008 );
3009
3010 Ok(ObjectLockDiagGuard::new(
3011 guard,
3012 diag_enabled,
3013 op,
3014 diag_enabled.then(|| bucket.to_string()),
3015 diag_enabled.then(|| object.to_string()),
3016 owner,
3017 ObjectLockDiagMode::Write,
3018 )
3019 .with_attempt(attempt))
3020 }
3021
3022 async fn acquire_object_write_lock_if_needed(
3023 &self,
3024 op: &'static str,
3025 bucket: &str,
3026 object: &str,
3027 opts: &mut ObjectOptions,
3028 ) -> Result<Option<ObjectLockDiagGuard>> {
3029 if opts.no_lock {
3030 return Ok(None);
3031 }
3032
3033 let guard = self.acquire_object_write_lock(op, bucket, object).await?;
3034 if let Some(signal) = guard.lock_lost_signal() {
3035 opts.add_namespace_lock_lost_signal(signal);
3036 }
3037 opts.ensure_namespace_lock_fence();
3038 opts.no_lock = true;
3039
3040 Ok(Some(guard))
3041 }
3042
3043 pub async fn acquire_restore_accept_guard(&self, bucket: &str, object: &str) -> Result<RestoreAcceptGuard> {
3053 let object = encode_dir_object(object);
3054 let guard = self
3055 .acquire_object_write_lock("restore_object_accept", bucket, &object)
3056 .await?;
3057 Ok(RestoreAcceptGuard(guard))
3058 }
3059
3060 pub async fn acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<RestoreWorkerGuard> {
3065 let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3066 let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3067 lock.get_write_lock_quiet(get_lock_acquire_timeout())
3068 .await
3069 .map(RestoreWorkerGuard)
3070 .map_err(|err| Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err))
3071 }
3072
3073 pub async fn try_acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<Option<RestoreWorkerGuard>> {
3077 let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3078 let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3079 match lock.get_write_lock_quiet(RESTORE_WORKER_LOCK_PROBE_TIMEOUT).await {
3080 Ok(guard) => Ok(Some(RestoreWorkerGuard(guard))),
3081 Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => Ok(None),
3082 Err(err) => Err(Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err)),
3083 }
3084 }
3085
3086 async fn acquire_delete_objects_write_locks(
3087 &self,
3088 bucket: &str,
3089 objects: &[ObjectToDelete],
3090 opts: &mut ObjectOptions,
3091 ) -> Result<Vec<ObjectLockDiagGuard>> {
3092 if opts.no_lock || objects.is_empty() {
3093 return Ok(Vec::new());
3094 }
3095
3096 let object_names = sorted_unique_delete_object_names(objects);
3097 let mut guards = Vec::with_capacity(object_names.len());
3100 for object in object_names {
3101 guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
3102 }
3103 opts.no_lock = true;
3104 for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
3105 opts.add_namespace_lock_lost_signal(signal);
3106 }
3107 opts.ensure_namespace_lock_fence();
3108
3109 Ok(guards)
3110 }
3111
3112 async fn acquire_object_read_lock_if_needed(
3113 &self,
3114 op: &'static str,
3115 bucket: &str,
3116 object: &str,
3117 opts: &mut ObjectOptions,
3118 ) -> Result<Option<ObjectLockDiagGuard>> {
3119 if opts.no_lock {
3120 return Ok(None);
3121 }
3122
3123 let diag_enabled = is_object_lock_diag_enabled();
3124 let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
3125 let acquire_start = Instant::now();
3126 let timeout = get_lock_acquire_timeout();
3127 let mut attempt = ObjectLockAttempt::start(op, bucket, object, opts.version_id.as_deref(), &ns_lock, "read", timeout);
3128 let result = ns_lock.get_read_lock(timeout).await;
3129 attempt.observe(&result);
3130 let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3131 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3132 log_object_lock_acquire_if_slow(
3133 op,
3134 bucket,
3135 object,
3136 owner.as_deref(),
3137 ObjectLockDiagMode::Read,
3138 acquire_start.elapsed(),
3139 diag_enabled,
3140 );
3141 opts.no_lock = true;
3142 opts.metadata_cache_safe = true;
3143
3144 Ok(Some(
3145 ObjectLockDiagGuard::new(
3146 guard,
3147 diag_enabled,
3148 op,
3149 diag_enabled.then(|| bucket.to_string()),
3150 diag_enabled.then(|| object.to_string()),
3151 owner,
3152 ObjectLockDiagMode::Read,
3153 )
3154 .with_attempt(attempt),
3155 ))
3156 }
3157
3158 pub async fn save_scanner_pause_backlog_replica(
3162 self: &Arc<Self>,
3163 pool_index: usize,
3164 set_index: usize,
3165 data: Vec<u8>,
3166 preconditions: crate::storage_api_contracts::object::HTTPPreconditions,
3167 ) -> Result<ObjectInfo> {
3168 let set = self
3169 .pools
3170 .get(pool_index)
3171 .and_then(|pool| pool.disk_set.get(set_index))
3172 .ok_or_else(|| Error::InvalidArgument("scanner-backlog".into(), "replica".into(), "unknown pool or set".into()))?;
3173 let set = Arc::clone(set);
3174 let store = Arc::clone(self);
3175 let write = async move {
3176 let object = "buckets/.scanner-pause-backlog.json";
3177 let mut opts = ObjectOptions::default();
3178 let object_guard = if store.single_pool() {
3181 None
3182 } else {
3183 Some(
3184 store
3185 .acquire_object_write_lock("scanner_backlog_replica", RUSTFS_META_BUCKET, object)
3186 .await?,
3187 )
3188 };
3189 let capacity_guard = if let Some(guard) = object_guard.as_ref() {
3190 guard.add_namespace_lock_fence(&mut opts);
3191 opts.no_lock = match store.pools.first().and_then(|pool| pool.disk_set.first()) {
3192 Some(fixed) => fixed.shares_namespace_lock_domain(&set).await,
3193 None => false,
3194 };
3195 let capacity_guard = store
3196 .acquire_external_decommission_capacity_fence(&[pool_index], DecommissionCapacityAdmission::ScannerBacklog)
3197 .await?;
3198 opts.add_namespace_lock_guard(&capacity_guard);
3199 Some(capacity_guard)
3200 } else {
3201 None
3202 };
3203 let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
3204 set,
3205 data,
3206 preconditions,
3207 opts,
3208 "publish",
3209 )
3210 .await;
3211 drop(capacity_guard);
3212 drop(object_guard);
3213 result
3214 };
3215 tokio::spawn(write).await.map_err(Error::from)?
3218 }
3219
3220 pub(super) async fn run_external_decommission_capacity_object_mutation<T, F, Fut>(
3221 &self,
3222 target_pool_idx: usize,
3223 bucket: &str,
3224 lock_object: &str,
3225 target_object: &str,
3226 opts: ObjectOptions,
3227 operation: F,
3228 ) -> Result<T>
3229 where
3230 F: FnOnce(ObjectOptions) -> Fut,
3231 Fut: std::future::Future<Output = Result<T>>,
3232 {
3233 self.run_external_decommission_capacity_object_operation(
3234 target_pool_idx,
3235 bucket,
3236 (lock_object, target_object),
3237 opts,
3238 false,
3239 operation,
3240 )
3241 .await
3242 }
3243
3244 pub(super) async fn run_external_decommission_capacity_object_delete<T, F, Fut>(
3245 &self,
3246 target_pool_idx: usize,
3247 bucket: &str,
3248 lock_object: &str,
3249 target_object: &str,
3250 opts: ObjectOptions,
3251 operation: F,
3252 ) -> Result<T>
3253 where
3254 F: FnOnce(ObjectOptions) -> Fut,
3255 Fut: std::future::Future<Output = Result<T>>,
3256 {
3257 let capacity_releasing = delete_only_releases_capacity(&opts);
3258 self.run_external_decommission_capacity_object_operation(
3259 target_pool_idx,
3260 bucket,
3261 (lock_object, target_object),
3262 opts,
3263 capacity_releasing,
3264 operation,
3265 )
3266 .await
3267 }
3268
3269 async fn run_external_decommission_capacity_object_operation<T, F, Fut>(
3270 &self,
3271 target_pool_idx: usize,
3272 bucket: &str,
3273 objects: (&str, &str),
3274 mut opts: ObjectOptions,
3275 capacity_releasing: bool,
3276 operation: F,
3277 ) -> Result<T>
3278 where
3279 F: FnOnce(ObjectOptions) -> Fut,
3280 Fut: std::future::Future<Output = Result<T>>,
3281 {
3282 let (lock_object, target_object) = objects;
3283 if self.single_pool() {
3284 return operation(opts).await;
3285 }
3286 let (capacity_guard, has_active_decommission) = if capacity_releasing {
3287 self.acquire_decommission_capacity_release_fence_with_active_source().await?
3288 } else {
3289 self.acquire_external_decommission_capacity_fence_with_active_source(
3290 &[target_pool_idx],
3291 DecommissionCapacityAdmission::Mutation,
3292 )
3293 .await?
3294 };
3295 let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3296 drop(capacity_guard);
3299 #[cfg(test)]
3300 crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3301 let guard = self
3302 .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3303 .await?;
3304 self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
3305 .await;
3306 let capacity_guard = if capacity_releasing {
3307 self.acquire_decommission_capacity_release_fence_with_active_source().await?.0
3308 } else {
3309 self.acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Mutation)
3310 .await?
3311 };
3312 (capacity_guard, Some(guard))
3313 } else {
3314 (capacity_guard, None)
3315 };
3316
3317 let result = operation(opts).await;
3318 drop(capacity_guard);
3319 drop(object_guard);
3320 result
3321 }
3322
3323 pub(crate) async fn acquire_external_decommission_commit_guards(
3334 &self,
3335 target_pool_idx: usize,
3336 bucket: &str,
3337 object: &str,
3338 no_lock: bool,
3339 admission: DecommissionCapacityAdmission,
3340 ) -> Result<(Option<ObjectLockDiagGuard>, bool, Option<rustfs_lock::NamespaceLockGuard>)> {
3341 let (capacity_guard, has_active_decommission) = self
3342 .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], admission)
3343 .await?;
3344 if !has_active_decommission {
3345 #[cfg(test)]
3348 {
3349 crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id);
3350 crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await;
3351 }
3352 return Ok((None, false, Some(capacity_guard)));
3353 }
3354 drop(capacity_guard);
3355 if no_lock {
3356 return Ok((None, false, None));
3357 }
3358
3359 let object_guard = self
3364 .acquire_object_write_lock("external_capacity_order", bucket, object)
3365 .await?;
3366 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3367 let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3368 let target_lock_covered = match (fixed_set, target_set) {
3369 (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3370 _ => false,
3371 };
3372 Ok((Some(object_guard), target_lock_covered, None))
3373 }
3374
3375 pub(super) async fn run_external_decommission_capacity_heal<T, F, Fut>(
3376 &self,
3377 target_pool_idx: usize,
3378 bucket: &str,
3379 lock_object: &str,
3380 mut opts: HealOpts,
3381 operation: F,
3382 ) -> Result<T>
3383 where
3384 F: FnOnce(HealOpts) -> Fut,
3385 Fut: std::future::Future<Output = Result<T>>,
3386 {
3387 if self.single_pool() {
3388 return operation(opts).await;
3389 }
3390 let (capacity_guard, has_active_decommission) = self
3391 .acquire_external_decommission_capacity_fence_with_active_source(
3392 &[target_pool_idx],
3393 DecommissionCapacityAdmission::Heal,
3394 )
3395 .await?;
3396 let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3397 drop(capacity_guard);
3400 #[cfg(test)]
3401 crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3402 let guard = self
3403 .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3404 .await?;
3405 let target_set = self
3406 .pools
3407 .get(target_pool_idx)
3408 .ok_or_else(|| Error::other("heal target pool is unavailable"))?
3409 .get_disks_for_heal_object(lock_object, &opts)?;
3410 let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) {
3411 Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await,
3412 None => false,
3413 };
3414 let capacity_guard = self
3415 .acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Heal)
3416 .await?;
3417 opts.no_lock = target_lock_covered;
3418 (capacity_guard, Some(guard))
3419 } else {
3420 (capacity_guard, None)
3421 };
3422
3423 #[cfg(test)]
3424 if !opts.no_lock {
3425 crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
3426 }
3427 let result = operation(opts).await;
3428 drop(capacity_guard);
3429 drop(object_guard);
3430 result
3431 }
3432
3433 pub(crate) async fn acquire_decommission_object_mutation_fence(
3434 &self,
3435 bucket: &str,
3436 object: &str,
3437 ) -> Result<DecommissionFixedReadAnchor> {
3438 if self.ctx.lock_manager().is_disabled() {
3439 return Err(Error::other("decommission object migration requires namespace locking"));
3440 }
3441
3442 #[cfg(test)]
3443 let test_namespace_lock_fence =
3444 decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration);
3445 let object = encode_dir_object(object);
3446 let mut opts = ObjectOptions::default();
3447 let guard = self
3448 .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts)
3449 .await?
3450 .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?;
3451 #[cfg(test)]
3452 let guard = {
3453 let mut guard = guard;
3454 guard.test_namespace_lock_fence = test_namespace_lock_fence;
3455 guard
3456 };
3457 Ok(DecommissionFixedReadAnchor { guard })
3458 }
3459
3460 pub(super) async fn apply_decommission_target_mutation_fence(
3461 &self,
3462 target_pool_idx: usize,
3463 object: &str,
3464 opts: &mut ObjectOptions,
3465 mutation_fence: Option<&ObjectLockDiagGuard>,
3466 ) {
3467 let Some(mutation_fence) = mutation_fence else {
3468 return;
3469 };
3470
3471 mutation_fence.add_namespace_lock_fence(opts);
3472 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3473 let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3474 opts.no_lock = match (fixed_set, target_set) {
3475 (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3476 _ => false,
3477 };
3478 }
3479
3480 pub(crate) async fn acquire_decommission_source_cleanup_fence(
3481 &self,
3482 bucket: &str,
3483 object: &str,
3484 source_set: &SetDisks,
3485 ) -> Result<SourceCleanupMutationFence> {
3486 if self.ctx.lock_manager().is_disabled() {
3487 return Err(Error::other("decommission source cleanup requires namespace locking"));
3488 }
3489
3490 #[cfg(test)]
3491 crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object);
3492 #[cfg(test)]
3493 let test_namespace_lock_fence =
3494 decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup);
3495 let object = encode_dir_object(object);
3496 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3497 let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await;
3498 let guard = self
3501 .acquire_object_write_lock("decommission_source_cleanup", bucket, &object)
3502 .await?;
3503 #[cfg(test)]
3504 let guard = {
3505 let mut guard = guard;
3506 guard.test_namespace_lock_fence = test_namespace_lock_fence;
3507 guard
3508 };
3509
3510 Ok(SourceCleanupMutationFence {
3511 guard,
3512 source_lock_covered,
3513 })
3514 }
3515
3516 pub(crate) async fn acquire_all_object_read_locks(
3517 &self,
3518 op: &'static str,
3519 bucket: &str,
3520 object: &str,
3521 opts: &mut ObjectOptions,
3522 ) -> Result<Vec<ObjectLockDiagGuard>> {
3523 let diag_enabled = is_object_lock_diag_enabled();
3524 let mut guards = Vec::with_capacity(self.pools.len() + 1);
3525
3526 let distributed = self.ctx.is_dist_erasure().await;
3530 if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3531 guards.push(guard);
3532 }
3533 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3534 let mut locked_sets = vec![fixed_set];
3535
3536 for pool in &self.pools {
3537 let hashed_set = pool.get_disks_by_key(object);
3538 let mut lock_domain_already_held = !distributed;
3539 if !lock_domain_already_held {
3540 for locked_set in &locked_sets {
3541 if locked_set.shares_namespace_lock_domain(&hashed_set).await {
3542 lock_domain_already_held = true;
3543 break;
3544 }
3545 }
3546 }
3547 if lock_domain_already_held {
3548 continue;
3549 }
3550 let ns_lock = hashed_set.new_ns_lock(bucket, object).await?;
3551 let acquire_start = Instant::now();
3552 let guard = ns_lock
3553 .get_read_lock(get_lock_acquire_timeout())
3554 .await
3555 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3556 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3557 log_object_lock_acquire_if_slow(
3558 op,
3559 bucket,
3560 object,
3561 owner.as_deref(),
3562 ObjectLockDiagMode::Read,
3563 acquire_start.elapsed(),
3564 diag_enabled,
3565 );
3566 guards.push(ObjectLockDiagGuard::new(
3567 guard,
3568 diag_enabled,
3569 op,
3570 diag_enabled.then(|| bucket.to_string()),
3571 diag_enabled.then(|| object.to_string()),
3572 owner,
3573 ObjectLockDiagMode::Read,
3574 ));
3575 locked_sets.push(hashed_set);
3576 }
3577 Ok(guards)
3578 }
3579
3580 pub(crate) async fn acquire_all_physical_object_read_locks(
3586 &self,
3587 op: &'static str,
3588 bucket: &str,
3589 object: &str,
3590 opts: &mut ObjectOptions,
3591 ) -> Result<Vec<ObjectLockDiagGuard>> {
3592 let diag_enabled = is_object_lock_diag_enabled();
3593 let distributed = self.ctx.is_dist_erasure().await;
3594 let mut guards = Vec::new();
3595 if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3596 guards.push(guard);
3597 }
3598
3599 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3600 let mut locked_sets = vec![fixed_set];
3601 for pool in &self.pools {
3602 for set in &pool.disk_set {
3603 let mut lock_domain_already_held = !distributed;
3604 if !lock_domain_already_held {
3605 for locked_set in &locked_sets {
3606 if locked_set.shares_namespace_lock_domain(set).await {
3607 lock_domain_already_held = true;
3608 break;
3609 }
3610 }
3611 }
3612 if lock_domain_already_held {
3613 continue;
3614 }
3615 let ns_lock = set.new_ns_lock(bucket, object).await?;
3616 let acquire_start = Instant::now();
3617 let guard = ns_lock
3618 .get_read_lock(get_lock_acquire_timeout())
3619 .await
3620 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3621 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3622 log_object_lock_acquire_if_slow(
3623 op,
3624 bucket,
3625 object,
3626 owner.as_deref(),
3627 ObjectLockDiagMode::Read,
3628 acquire_start.elapsed(),
3629 diag_enabled,
3630 );
3631 guards.push(ObjectLockDiagGuard::new(
3632 guard,
3633 diag_enabled,
3634 op,
3635 diag_enabled.then(|| bucket.to_string()),
3636 diag_enabled.then(|| object.to_string()),
3637 owner,
3638 ObjectLockDiagMode::Read,
3639 ));
3640 locked_sets.push(Arc::clone(set));
3641 }
3642 }
3643 Ok(guards)
3644 }
3645
3646 pub(crate) async fn acquire_all_physical_object_write_locks(
3647 &self,
3648 op: &'static str,
3649 bucket: &str,
3650 object: &str,
3651 ) -> Result<Vec<ObjectLockDiagGuard>> {
3652 self.acquire_physical_object_write_locks(op, bucket, object, false).await
3653 }
3654
3655 async fn acquire_remaining_physical_object_write_locks(
3656 &self,
3657 op: &'static str,
3658 bucket: &str,
3659 object: &str,
3660 ) -> Result<Vec<ObjectLockDiagGuard>> {
3661 self.acquire_physical_object_write_locks(op, bucket, object, true).await
3662 }
3663
3664 async fn acquire_physical_object_write_locks(
3665 &self,
3666 op: &'static str,
3667 bucket: &str,
3668 object: &str,
3669 store_lock_already_held: bool,
3670 ) -> Result<Vec<ObjectLockDiagGuard>> {
3671 if self.ctx.lock_manager().is_disabled() {
3677 return Err(Error::other("physical object mutation requires namespace locking"));
3678 }
3679 let diag_enabled = is_object_lock_diag_enabled();
3680 let distributed = self.ctx.is_dist_erasure().await;
3681 let mut guards = Vec::new();
3682 if !store_lock_already_held {
3683 guards.push(self.acquire_object_write_lock(op, bucket, object).await?);
3684 }
3685 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3686 let mut locked_sets = vec![fixed_set];
3687 for pool in &self.pools {
3688 for set in &pool.disk_set {
3689 let mut lock_domain_already_held = !distributed;
3690 if !lock_domain_already_held {
3691 for locked_set in &locked_sets {
3692 if locked_set.shares_namespace_lock_domain(set).await {
3693 lock_domain_already_held = true;
3694 break;
3695 }
3696 }
3697 }
3698 if lock_domain_already_held {
3699 continue;
3700 }
3701 let ns_lock = set.new_ns_lock(bucket, object).await?;
3702 let acquire_start = Instant::now();
3703 let guard = ns_lock
3704 .get_write_lock(get_lock_acquire_timeout())
3705 .await
3706 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3707 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3708 log_object_lock_acquire_if_slow(
3709 op,
3710 bucket,
3711 object,
3712 owner.as_deref(),
3713 ObjectLockDiagMode::Write,
3714 acquire_start.elapsed(),
3715 diag_enabled,
3716 );
3717 guards.push(ObjectLockDiagGuard::new(
3718 guard,
3719 diag_enabled,
3720 op,
3721 diag_enabled.then(|| bucket.to_string()),
3722 diag_enabled.then(|| object.to_string()),
3723 owner,
3724 ObjectLockDiagMode::Write,
3725 ));
3726 locked_sets.push(Arc::clone(set));
3727 }
3728 }
3729 Ok(guards)
3730 }
3731
3732 pub(crate) async fn acquire_data_movement_publication_write_locks(
3733 &self,
3734 bucket: &str,
3735 object: &str,
3736 source_pool_idx: usize,
3737 target_pool_idx: usize,
3738 include_fixed_domain: bool,
3739 ) -> Result<Vec<ObjectLockDiagGuard>> {
3740 if self.ctx.lock_manager().is_disabled() {
3741 return Err(Error::other("data movement publication requires namespace locking"));
3742 }
3743 let distributed = self.ctx.is_dist_erasure().await;
3744 let diag_enabled = is_object_lock_diag_enabled();
3745 let mut pool_indices = [source_pool_idx, target_pool_idx];
3746 pool_indices.sort_unstable();
3747 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3748 let mut locked_sets = Vec::with_capacity(3);
3749 let mut guards = Vec::with_capacity(3);
3750 #[cfg(test)]
3751 let test_namespace_lock_fence = decommission_mutation_fence_for_test(
3752 bucket,
3753 &decode_dir_object(object),
3754 DecommissionMutationFenceTestPhase::Migration,
3755 );
3756
3757 if include_fixed_domain {
3762 let guard = self
3763 .acquire_object_write_lock("data_movement_publication", bucket, object)
3764 .await?;
3765 #[cfg(test)]
3766 let guard = {
3767 let mut guard = guard;
3768 guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3769 guard
3770 };
3771 guards.push(guard);
3772 locked_sets.push(fixed_set);
3773 }
3774 for pool_idx in pool_indices {
3775 let pool = self
3776 .pools
3777 .get(pool_idx)
3778 .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
3779 let set = pool.get_disks_by_key(object);
3780 let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
3781 if !lock_domain_already_held {
3782 for locked_set in &locked_sets {
3783 if locked_set.shares_namespace_lock_domain(&set).await {
3784 lock_domain_already_held = true;
3785 break;
3786 }
3787 }
3788 }
3789 if lock_domain_already_held {
3790 continue;
3791 }
3792 let ns_lock = set.new_ns_lock(bucket, object).await?;
3793 let acquire_start = Instant::now();
3794 let guard = ns_lock
3795 .get_write_lock(get_lock_acquire_timeout())
3796 .await
3797 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3798 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3799 log_object_lock_acquire_if_slow(
3800 "data_movement_publication",
3801 bucket,
3802 object,
3803 owner.as_deref(),
3804 ObjectLockDiagMode::Write,
3805 acquire_start.elapsed(),
3806 diag_enabled,
3807 );
3808 let guard = ObjectLockDiagGuard::new(
3809 guard,
3810 diag_enabled,
3811 "data_movement_publication",
3812 diag_enabled.then(|| bucket.to_string()),
3813 diag_enabled.then(|| object.to_string()),
3814 owner,
3815 ObjectLockDiagMode::Write,
3816 );
3817 #[cfg(test)]
3818 let guard = {
3819 let mut guard = guard;
3820 if guards.is_empty() {
3821 guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3822 }
3823 guard
3824 };
3825 guards.push(guard);
3826 locked_sets.push(set);
3827 }
3828 Ok(guards)
3829 }
3830
3831 fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
3832 if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
3833 return reader;
3834 }
3835
3836 if let Some(guard) = guard {
3837 reader.stream = Box::new(LockGuardedReader {
3838 inner: reader.stream,
3839 guard: Some(guard),
3840 });
3841 }
3842
3843 reader
3844 }
3845
3846 async fn get_latest_accessible_object_info_with_idx(
3847 &self,
3848 bucket: &str,
3849 object: &str,
3850 opts: &ObjectOptions,
3851 ) -> Result<(ObjectInfo, usize)> {
3852 let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?;
3853 resolve_latest_object_access(bucket, object, info, idx, opts)
3854 }
3855
3856 pub(crate) async fn select_data_movement_pool_idx(
3857 &self,
3858 bucket: &str,
3859 object: &str,
3860 size: i64,
3861 opts: &ObjectOptions,
3862 no_lock: bool,
3863 ) -> Result<usize> {
3864 let capacity_owner = DecommissionCapacityOwner::from_options(opts);
3865 match self
3866 .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
3867 .await
3868 {
3869 Ok((pinfo, _)) => {
3870 if let Some(owner) = capacity_owner {
3871 if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
3872 return Ok(pinfo.index);
3873 }
3874 } else {
3875 return Ok(pinfo.index);
3876 }
3877 }
3878 Err(err) => {
3879 if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
3880 return Err(err);
3881 }
3882 }
3883 }
3884 if let Some(owner) = capacity_owner {
3885 let expected_data_bytes = opts
3886 .capacity_expected_data_bytes()
3887 .or_else(|| usize::try_from(size).ok())
3888 .unwrap_or_default();
3889 return self
3890 .select_decommission_capacity_target_pool(owner, expected_data_bytes)
3891 .await;
3892 }
3893
3894 self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
3895 }
3896
3897 async fn find_data_movement_target_info(
3898 &self,
3899 bucket: &str,
3900 object: &str,
3901 target_pool_idx: usize,
3902 opts: &ObjectOptions,
3903 ) -> Result<Option<ObjectInfo>> {
3904 let mut lookup_opts = version_aware_lookup_opts(opts, true);
3905 lookup_opts.include_part_checksums = true;
3906
3907 let Some(pool) = self.pools.get(target_pool_idx) else {
3908 return Err(Error::other(format!(
3909 "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
3910 )));
3911 };
3912
3913 match pool.get_object_info(bucket, object, &lookup_opts).await {
3914 Ok(info) => Ok(Some(info)),
3915 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
3916 Err(err) => Err(err),
3917 }
3918 }
3919
3920 async fn has_equivalent_data_movement_delete_marker(
3921 &self,
3922 bucket: &str,
3923 object: &str,
3924 source: &ObjectInfo,
3925 opts: &ObjectOptions,
3926 target_pool_idx: usize,
3927 ) -> Result<bool> {
3928 let equivalent = resolve_data_movement_delete_marker_resume_result(
3929 self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3930 .await,
3931 source,
3932 opts.src_pool_idx,
3933 target_pool_idx,
3934 )?;
3935 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3936 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3937 .await?;
3938 }
3939 Ok(equivalent)
3940 }
3941
3942 async fn has_equivalent_data_movement_tiered_object(
3943 &self,
3944 bucket: &str,
3945 object: &str,
3946 source: &rustfs_filemeta::FileInfo,
3947 opts: &ObjectOptions,
3948 target_pool_idx: usize,
3949 ) -> Result<bool> {
3950 let equivalent = resolve_data_movement_tiered_resume_result(
3951 self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3952 .await,
3953 source,
3954 opts.src_pool_idx,
3955 target_pool_idx,
3956 )?;
3957 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3958 let expected_data_bytes = usize::try_from(source.size).unwrap_or_default();
3959 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
3960 .await?;
3961 }
3962 Ok(equivalent)
3963 }
3964
3965 async fn has_equivalent_data_movement_tier_free_version(
3966 &self,
3967 bucket: &str,
3968 object: &str,
3969 source: &rustfs_filemeta::FileInfo,
3970 opts: &ObjectOptions,
3971 target_pool_idx: usize,
3972 ) -> Result<bool> {
3973 let pool = self
3974 .pools
3975 .get(target_pool_idx)
3976 .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
3977 let equivalent = pool
3978 .get_disks_by_key(object)
3979 .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
3980 .await?;
3981 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3982 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3983 .await?;
3984 }
3985 Ok(equivalent)
3986 }
3987
3988 fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
3989 result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
3990 }
3991
3992 fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
3993 result.map_err(|err| {
3994 crate::data_movement::data_movement_context_error(
3995 format!("failed to decommission tiered object for {bucket}/{object}: {err}"),
3996 err,
3997 )
3998 })
3999 }
4000
4001 #[instrument(skip(self, fi, opts))]
4002 pub(crate) async fn decommission_tiered_object(
4003 self: &Arc<Self>,
4004 bucket: &str,
4005 object: &str,
4006 fi: &rustfs_filemeta::FileInfo,
4007 opts: &ObjectOptions,
4008 ) -> Result<()> {
4009 check_put_object_args(bucket, object)?;
4010
4011 let mut opts = opts.clone();
4012 let is_free_version = fi.tier_free_version();
4013 if is_free_version {
4014 opts.incl_free_versions = true;
4015 }
4016 let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
4017 None
4018 } else {
4019 let expected = opts
4020 .expected_bucket_incarnation_id
4021 .ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
4022 let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
4023 if let Some(namespace_guard) = guard.namespace_lock_guard() {
4024 opts.add_bucket_lifecycle_lock_guard(namespace_guard);
4025 }
4026 Some(guard)
4027 };
4028
4029 let publication_fence = self
4032 .acquire_tiered_remote_tuple_publication_fence(bucket, object, opts.src_pool_idx, fi)
4033 .await?;
4034
4035 let logical_object = object;
4036 let object = encode_dir_object(logical_object);
4037 ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4038 if self.single_pool() {
4039 return Self::resolve_decommission_tiered_object_result(
4040 Err(Error::other("single pool deployments cannot decommission tiered objects")),
4041 bucket,
4042 &object,
4043 );
4044 }
4045
4046 let idx = if uses_data_movement_pool_selection(&opts) {
4047 Self::resolve_decommission_target_pool_idx_result(
4048 self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
4049 .await,
4050 bucket,
4051 &object,
4052 )?
4053 } else {
4054 Self::resolve_decommission_target_pool_idx_result(
4055 self.get_pool_idx_no_lock(bucket, &object, fi.size).await,
4056 bucket,
4057 &object,
4058 )?
4059 };
4060 #[cfg(all(test, feature = "test-util"))]
4061 if is_free_version {
4062 pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
4063 }
4064 let publication_guard = publication_fence.into_commit_guard(idx, bucket, &object).await?;
4065 publication_guard.add_namespace_lock_fence(&mut opts);
4066 opts.no_lock = true;
4067 let mut fi = publication_guard
4068 .revalidated_tiered_source()
4069 .ok_or_else(|| Error::other("tiered data movement publication guard is missing its revalidated source"))?
4070 .clone();
4071 if opts.data_movement {
4072 crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
4073 }
4074 if opts.data_movement && idx == opts.src_pool_idx {
4075 let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4076 Some(
4077 self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default())
4078 .await?,
4079 )
4080 } else {
4081 self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
4082 .await
4083 };
4084 let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
4085 if is_free_version && target_pool_idx == opts.src_pool_idx {
4086 return Err(Error::DiskFull);
4087 }
4088 let equivalent = if is_free_version {
4089 self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
4090 .await?
4091 } else {
4092 self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
4093 .await?
4094 };
4095 if equivalent {
4096 return Ok(());
4097 }
4098
4099 return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
4100 }
4101
4102 let expected_data_bytes = usize::try_from(fi.size).ok();
4103 let result = self
4104 .run_decommission_capacity_admitted_mutation(
4105 idx,
4106 DecommissionCapacityOwner::from_options(&opts),
4107 expected_data_bytes,
4108 || async {
4109 if is_free_version {
4110 self.pools[idx]
4111 .get_disks_by_key(&object)
4112 .decommission_tier_free_version(bucket, &object, &fi, &opts)
4113 .await
4114 } else {
4115 self.pools[idx]
4116 .get_disks_by_key(&object)
4117 .decommission_tiered_object(bucket, &object, &fi, &opts)
4118 .await
4119 }
4120 },
4121 )
4122 .await;
4123 if matches!(result, Err(Error::PreconditionFailed)) {
4124 if self
4125 .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
4126 .await?
4127 {
4128 return Ok(());
4129 }
4130 return Err(StorageError::DataMovementOverwriteErr(
4131 bucket.to_owned(),
4132 object,
4133 opts.version_id.clone().unwrap_or_default(),
4134 ));
4135 }
4136 if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
4137 return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
4138 }
4139 Self::resolve_decommission_tiered_object_result(result, bucket, &object)
4140 }
4141
4142 pub async fn get_object_reader_for_copy(
4150 &self,
4151 bucket: &str,
4152 object: &str,
4153 range: Option<HTTPRangeSpec>,
4154 h: HeaderMap,
4155 opts: &ObjectOptions,
4156 ) -> Result<(GetObjectReader, tokio_util::sync::CancellationToken)> {
4157 let cancellation = tokio_util::sync::CancellationToken::new();
4158 let reader = crate::set_disk::with_get_object_read_cancellation(
4159 cancellation.clone(),
4160 crate::set_disk::with_get_object_read_policy(
4161 crate::set_disk::GetObjectReadPolicy::CopySource,
4162 self.handle_get_object_reader(bucket, object, range, h, opts),
4163 ),
4164 )
4165 .await?;
4166 Ok((reader, cancellation))
4167 }
4168
4169 #[instrument(level = "debug", skip(self, h))]
4170 #[hotpath::measure(impl_type = "ECStore")]
4171 pub(super) async fn handle_get_object_reader(
4172 &self,
4173 bucket: &str,
4174 object: &str,
4175 range: Option<HTTPRangeSpec>,
4176 h: HeaderMap,
4177 opts: &ObjectOptions,
4178 ) -> Result<GetObjectReader> {
4179 check_get_obj_args(bucket, object)?;
4180
4181 let object = rustfs_utils::path::encode_dir_object_ref(object);
4182 let mut opts = opts.clone();
4183 let read_lock_guard = self
4184 .acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
4185 .await?;
4186
4187 let reader = if self.single_pool() {
4188 self.pools[0]
4189 .get_object_reader(bucket, object.as_ref(), range, h, &opts)
4190 .await?
4191 } else {
4192 Box::pin(async {
4195 let (metadata, idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
4196 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
4197 return Err(error);
4198 }
4199 self.pools[idx]
4200 .get_object_reader_with_prepared_metadata(bucket, object.as_ref(), range, h, &opts, metadata)
4201 .await
4202 })
4203 .await?
4204 };
4205
4206 Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
4207 }
4208
4209 async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
4210 check_put_object_args(bucket, object)?;
4211
4212 let object = encode_dir_object(object);
4213 let mut opts = opts.clone();
4214 if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
4215 opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
4216 }
4217 if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
4218 let expected_incarnation_id = opts
4219 .expected_bucket_incarnation_id
4220 .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
4221 if opts.object_lock_config_snapshot.is_none() {
4222 opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
4223 }
4224 let snapshot = match opts.object_lock_config_snapshot.as_ref() {
4225 Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
4226 Arc::clone(snapshot)
4227 }
4228 _ => {
4229 return Err(Error::other(
4230 "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
4231 ));
4232 }
4233 };
4234 snapshot.add_lock_fences(&mut opts);
4235 }
4236 Ok((object, opts))
4237 }
4238
4239 async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
4240 if self.single_pool() {
4241 return Ok(0);
4242 }
4243
4244 let idx = if uses_data_movement_pool_selection(opts) {
4245 self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
4246 } else if opts.no_lock {
4247 self.get_pool_idx_no_lock(bucket, object, size).await?
4248 } else {
4249 self.get_pool_idx(bucket, object, size).await?
4250 };
4251
4252 if opts.data_movement && idx == opts.src_pool_idx {
4253 return Err(StorageError::DataMovementOverwriteErr(
4254 bucket.to_owned(),
4255 object.to_owned(),
4256 opts.version_id.clone().unwrap_or_default(),
4257 ));
4258 }
4259 Ok(idx)
4260 }
4261
4262 #[cfg(test)]
4263 pub(crate) async fn put_object_for_data_movement(
4264 self: &Arc<Self>,
4265 bucket: &str,
4266 object: &str,
4267 data: &mut PutObjReader,
4268 opts: &ObjectOptions,
4269 mutation_fence: Option<&ObjectLockDiagGuard>,
4270 ) -> Result<(usize, Result<ObjectInfo>)> {
4271 self.put_object_for_data_movement_inner(bucket, object, data, opts, mutation_fence, None)
4272 .await
4273 }
4274
4275 pub(crate) async fn put_object_for_data_movement_with_publication_fence(
4276 self: &Arc<Self>,
4277 bucket: &str,
4278 object: &str,
4279 data: &mut PutObjReader,
4280 opts: &ObjectOptions,
4281 publication_fence: RemoteTuplePublicationFence,
4282 ) -> Result<(usize, Result<ObjectInfo>)> {
4283 self.put_object_for_data_movement_inner(bucket, object, data, opts, None, Some(publication_fence))
4284 .await
4285 }
4286
4287 async fn put_object_for_data_movement_inner(
4288 self: &Arc<Self>,
4289 bucket: &str,
4290 object: &str,
4291 data: &mut PutObjReader,
4292 opts: &ObjectOptions,
4293 mutation_fence: Option<&ObjectLockDiagGuard>,
4294 publication_fence: Option<RemoteTuplePublicationFence>,
4295 ) -> Result<(usize, Result<ObjectInfo>)> {
4296 if !opts.data_movement {
4297 return Err(Error::other("data movement PUT requires data_movement options"));
4298 }
4299 let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
4300 ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4301 let idx = self
4302 .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4303 .await?;
4304 let fixed_read_anchor = publication_fence
4305 .as_ref()
4306 .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
4307 self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
4308 .await;
4309 let expected_data_bytes = usize::try_from(data.size()).ok();
4310 let result = self
4311 .run_decommission_capacity_admitted_mutation_with_capacity_lease(
4312 idx,
4313 DecommissionCapacityOwner::from_options(&opts),
4314 expected_data_bytes,
4315 |capacity_lease| async move {
4316 if let Some(capacity_lease) = capacity_lease {
4317 opts.add_namespace_lock_lost_signal(capacity_lease);
4318 }
4319 let result = match publication_fence {
4320 Some(publication_fence) => {
4321 self.pools[idx]
4322 .put_object_with_old_current_size_for_data_movement(
4323 bucket,
4324 &object,
4325 data,
4326 &opts,
4327 publication_fence,
4328 )
4329 .await
4330 }
4331 None => {
4332 self.pools[idx]
4333 .put_object_with_old_current_size(bucket, &object, data, &opts)
4334 .await
4335 }
4336 };
4337 result.map(|(object_info, _)| object_info)
4338 },
4339 )
4340 .await;
4341 let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
4342 if result.is_ok() {
4343 list_objects::observe_list_objects_mutation(self, bucket).await;
4344 }
4345 Ok((idx, result))
4346 }
4347
4348 #[instrument(level = "debug", skip(self, data))]
4349 #[hotpath::measure(impl_type = "ECStore")]
4350 pub(super) async fn handle_put_object(
4351 &self,
4352 bucket: &str,
4353 object: &str,
4354 data: &mut PutObjReader,
4355 opts: &ObjectOptions,
4356 ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
4357 let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
4358 let idx = self
4359 .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4360 .await?;
4361 let mut opts = opts;
4362 if !self.single_pool() {
4363 opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4364 }
4365 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4366 let result = self.pools[idx]
4367 .put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
4368 .await;
4369 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4370 result
4371 }
4372
4373 #[instrument(level = "trace", skip(self))]
4374 pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4375 self.get_object_info_snapshot(bucket, object, opts, false).await
4376 }
4377
4378 #[instrument(level = "trace", skip_all)]
4384 pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4385 self.get_object_info_snapshot(bucket, object, opts, true).await
4386 }
4387
4388 async fn get_object_info_snapshot(
4389 &self,
4390 bucket: &str,
4391 object: &str,
4392 opts: &ObjectOptions,
4393 allow_delete_marker: bool,
4394 ) -> Result<ObjectInfo> {
4395 check_object_args(bucket, object)?;
4396
4397 let object = encode_dir_object(object);
4398 let mut opts = opts.clone();
4399 let _object_lock_guard = self
4400 .acquire_object_read_lock_if_needed("get_object_info", bucket, &object, &mut opts)
4401 .await?;
4402
4403 let info = if self.single_pool() {
4404 self.pools[0].get_object_info(bucket, object.as_str(), &opts).await?
4405 } else if allow_delete_marker {
4406 self.get_latest_object_info_with_idx(bucket, object.as_str(), &opts).await?.0
4407 } else {
4408 self.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
4409 .await?
4410 .0
4411 };
4412 opts.precondition_check(&info)?;
4413 Ok(info)
4414 }
4415
4416 #[instrument(skip(self))]
4417 #[allow(clippy::too_many_arguments)]
4418 pub(super) async fn handle_copy_object(
4419 &self,
4420 src_bucket: &str,
4421 src_object: &str,
4422 dst_bucket: &str,
4423 dst_object: &str,
4424 src_info: &mut ObjectInfo,
4425 src_opts: &ObjectOptions,
4426 dst_opts: &ObjectOptions,
4427 ) -> Result<ObjectInfo> {
4428 check_copy_obj_args(src_bucket, src_object)?;
4429 check_copy_obj_args(dst_bucket, dst_object)?;
4430
4431 let src_object = encode_dir_object(src_object);
4432 let dst_object = encode_dir_object(dst_object);
4433
4434 let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
4435
4436 let mut dst_opts = dst_opts.clone();
4437 dst_opts.inherit_shard_integrity(src_info);
4438 if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
4439 dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
4440 }
4441 let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
4442 None
4443 } else {
4444 Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
4445 };
4446 let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4447 dst_opts.add_bucket_lifecycle_lock_guard(guard);
4448 let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
4449 if dst_opts
4450 .expected_bucket_incarnation_id
4451 .is_some_and(|expected| expected != current_incarnation_id)
4452 {
4453 return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
4454 }
4455 Some(current_incarnation_id)
4456 } else {
4457 dst_opts.expected_bucket_incarnation_id
4458 };
4459 if dst_opts
4460 .bucket_lifecycle_lock_fence
4461 .as_ref()
4462 .is_some_and(NamespaceLockFence::is_lock_lost)
4463 {
4464 return Err(StorageError::NamespaceLockQuorumUnavailable {
4465 mode: "copy_object_bucket_generation",
4466 bucket: dst_bucket.to_string(),
4467 object: dst_object.clone(),
4468 required: 1,
4469 achieved: 0,
4470 });
4471 }
4472 if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) {
4473 let incarnation_id =
4474 current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?;
4475 let lifecycle_fence = dst_opts
4476 .bucket_lifecycle_lock_fence
4477 .as_ref()
4478 .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?;
4479 let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() {
4480 Some(snapshot) => Arc::clone(snapshot),
4481 None => {
4482 self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence)
4483 .await?
4484 }
4485 };
4486 if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) {
4487 return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation"));
4488 }
4489 snapshot.add_lock_fences(&mut dst_opts);
4490 dst_opts.object_lock_config_snapshot = Some(snapshot);
4491 }
4492 let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() {
4493 self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts)
4494 .await?
4495 } else {
4496 None
4497 };
4498
4499 if cp_src_dst_same {
4500 let (_, pool_idx) = self
4501 .get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
4502 .await?;
4503 let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
4504
4505 if source_pool_writable
4506 && let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
4507 && src_vid == dst_vid
4508 {
4509 let capacity_object = dst_object.clone();
4510 return self
4511 .run_external_decommission_capacity_object_mutation(
4512 pool_idx,
4513 dst_bucket,
4514 &capacity_object,
4515 &capacity_object,
4516 dst_opts.clone(),
4517 |opts| async move {
4518 self.pools[pool_idx]
4519 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4520 .await
4521 },
4522 )
4523 .await;
4524 }
4525
4526 if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
4527 if src_info.metadata_only {
4528 let capacity_object = dst_object.clone();
4538 return self
4539 .run_external_decommission_capacity_object_mutation(
4540 pool_idx,
4541 dst_bucket,
4542 &capacity_object,
4543 &capacity_object,
4544 dst_opts.clone(),
4545 |opts| async move {
4546 self.pools[pool_idx]
4547 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4548 .await
4549 },
4550 )
4551 .await;
4552 }
4553 let mut put_opts = ObjectOptions {
4555 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4556 versioned: dst_opts.versioned,
4557 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4558 version_id: dst_opts.version_id.clone(),
4559 no_lock: dst_opts.no_lock,
4560 mod_time: dst_opts.mod_time,
4561 http_preconditions: dst_opts.http_preconditions.clone(),
4562 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4563 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4564 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4565 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4566 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4567 ..Default::default()
4568 };
4569 if !self.single_pool() {
4570 put_opts.decommission_capacity_admission =
4571 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4572 }
4573 return if let Some(reader) = src_info.put_object_reader.as_mut() {
4574 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4575 let result = self.pools[pool_idx]
4576 .put_object(dst_bucket, &dst_object, reader, &put_opts)
4577 .await;
4578 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4579 result
4580 } else {
4581 Err(StorageError::InvalidArgument(
4582 src_bucket.to_owned(),
4583 src_object.to_owned(),
4584 "put_object_reader is none".to_owned(),
4585 ))
4586 };
4587 }
4588
4589 if source_pool_writable && dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
4590 if let Some(reader) = src_info.put_object_reader.as_mut() {
4596 let mut put_opts = ObjectOptions {
4597 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4598 versioned: dst_opts.versioned,
4599 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4600 version_id: dst_opts.version_id.clone(),
4601 no_lock: dst_opts.no_lock,
4602 mod_time: dst_opts.mod_time,
4603 http_preconditions: dst_opts.http_preconditions.clone(),
4604 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4605 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4606 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4607 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4608 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4609 ..Default::default()
4610 };
4611 if !self.single_pool() {
4612 put_opts.decommission_capacity_admission =
4613 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4614 }
4615 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4616 let result = self.pools[pool_idx]
4617 .put_object(dst_bucket, &dst_object, reader, &put_opts)
4618 .await;
4619 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4620 return result;
4621 }
4622 src_info.version_only = true;
4623 let capacity_object = dst_object.clone();
4624 return self
4625 .run_external_decommission_capacity_object_mutation(
4626 pool_idx,
4627 dst_bucket,
4628 &capacity_object,
4629 &capacity_object,
4630 dst_opts.clone(),
4631 |opts| async move {
4632 self.pools[pool_idx]
4633 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4634 .await
4635 },
4636 )
4637 .await;
4638 }
4639 }
4640
4641 let pool_idx = if dst_opts.no_lock {
4642 self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
4643 } else {
4644 self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
4645 };
4646 let dst_object_name = dst_object.as_str();
4647
4648 let mut put_opts = ObjectOptions {
4649 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4650 versioned: dst_opts.versioned,
4651 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4652 version_id: dst_opts.version_id.clone(),
4653 no_lock: dst_opts.no_lock,
4654 mod_time: dst_opts.mod_time,
4655 http_preconditions: dst_opts.http_preconditions.clone(),
4656 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4657 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4658 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4659 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4660 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4661 ..Default::default()
4662 };
4663 if !self.single_pool() {
4664 put_opts.decommission_capacity_admission =
4665 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4666 }
4667
4668 if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
4669 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4670 let result = self.pools[pool_idx]
4671 .put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
4672 .await;
4673 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4674 return result;
4675 }
4676
4677 Err(StorageError::InvalidArgument(
4678 src_bucket.to_owned(),
4679 src_object.to_owned(),
4680 "put_object_reader is none".to_owned(),
4681 ))
4682 }
4683
4684 pub(super) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> bool {
4691 let prefix = decode_dir_object(object);
4692 let mut purged = false;
4693 for pool in self.pools.iter() {
4694 for set in pool.disk_set.iter() {
4695 match set.purge_orphan_dir_object(bucket, &prefix).await {
4696 Ok(set_purged) => purged |= set_purged,
4697 Err(err) => {
4698 warn!(
4699 bucket,
4700 prefix,
4701 pool_index = pool.pool_idx,
4702 error = ?err,
4703 "failed to purge orphan directory prefix"
4704 );
4705 }
4706 }
4707 }
4708 }
4709 purged
4710 }
4711
4712 pub async fn delete_object_with_tier_delete_journal(
4713 self: &Arc<Self>,
4714 bucket: &str,
4715 object: &str,
4716 opts: ObjectOptions,
4717 ) -> Result<ObjectInfo> {
4718 let result = self
4719 .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
4720 .await;
4721 if result.is_ok() {
4722 list_objects::observe_list_objects_mutation(self, bucket).await;
4723 }
4724 result
4725 }
4726
4727 pub async fn delete_objects_with_tier_delete_journal(
4728 self: &Arc<Self>,
4729 bucket: &str,
4730 objects: Vec<ObjectToDelete>,
4731 opts: ObjectOptions,
4732 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
4733 let result = self
4734 .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
4735 .await;
4736 let success_count = result.1.iter().filter(|err| err.is_none()).count();
4737 if success_count > 0 {
4738 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4739 }
4740 result
4741 }
4742
4743 pub async fn delete_objects_with_tier_delete_journal_and_accounting(
4744 self: &Arc<Self>,
4745 bucket: &str,
4746 objects: Vec<ObjectToDelete>,
4747 opts: ObjectOptions,
4748 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
4749 let result = self
4750 .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
4751 .await;
4752 let success_count = result.1.iter().filter(|err| err.is_none()).count();
4753 if success_count > 0 {
4754 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4755 }
4756 result
4757 }
4758
4759 #[instrument(skip(self))]
4760 pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
4761 self.handle_delete_object_with_journal(bucket, object, opts, None).await
4762 }
4763
4764 pub(super) async fn handle_delete_object_with_journal(
4765 &self,
4766 bucket: &str,
4767 object: &str,
4768 opts: ObjectOptions,
4769 tier_journal_api: Option<Arc<ECStore>>,
4770 ) -> Result<ObjectInfo> {
4771 Box::pin(async move {
4772 let mut opts = opts;
4773 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4774 let result = self
4775 .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)
4776 .await;
4777 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4778 result
4779 })
4780 .await
4781 }
4782
4783 async fn handle_delete_object_with_journal_inner(
4784 &self,
4785 bucket: &str,
4786 object: &str,
4787 opts: ObjectOptions,
4788 tier_journal_api: Option<Arc<ECStore>>,
4789 ) -> Result<ObjectInfo> {
4790 check_del_obj_args(bucket, object)?;
4791
4792 if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
4793 return Err(Error::other("lifecycle delete-all requires namespace locking"));
4794 }
4795
4796 let _bucket_lifecycle_guard =
4797 if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
4798 None
4799 } else if opts.delete_prefix {
4800 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
4801 } else {
4802 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
4803 };
4804 let object = if opts.delete_prefix && !opts.delete_prefix_object {
4805 object.to_owned()
4806 } else {
4807 encode_dir_object(object)
4808 };
4809 let object = object.as_str();
4810 let mut opts = opts;
4811 ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
4812 let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
4813 Some(get_expiry_configs(self, bucket).await?)
4814 } else {
4815 None
4816 };
4817 opts.tier_delete_journal_api = tier_journal_api.clone();
4818 if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4819 opts.add_bucket_lifecycle_lock_guard(guard);
4820 }
4821
4822 if !is_meta_bucketname(bucket) {
4823 get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
4824 }
4825 let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
4826 Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
4827 } else {
4828 None
4829 };
4830 if let Some(guard) = _object_lock_metadata_guard.as_ref() {
4831 opts.add_namespace_lock_guard(guard);
4832 }
4833 let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
4834 let (state, incarnation_id, config_revision) =
4835 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
4836 opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
4837 self.id,
4838 bucket,
4839 incarnation_id,
4840 config_revision,
4841 state,
4842 )));
4843 Some(incarnation_id)
4844 } else {
4845 None
4846 };
4847 if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
4848 && expected != current
4849 {
4850 return Err(StorageError::BucketNotFound(bucket.to_string()));
4851 }
4852 if opts.expected_bucket_incarnation_id.is_none() {
4853 opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
4854 }
4855 #[cfg(any(test, feature = "test-util"))]
4856 pause_delete_after_object_lock_snapshot(bucket).await;
4857
4858 if opts.delete_prefix && !opts.delete_prefix_object {
4859 delete_recursive_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4860 return Ok(ObjectInfo::default());
4861 }
4862
4863 let _object_lock_guard = if opts.expected_current_version_id.is_none() {
4864 self.acquire_object_write_lock_if_needed("delete_object", bucket, object, &mut opts)
4865 .await?
4866 } else {
4867 None
4868 };
4869 #[cfg(any(test, feature = "test-util"))]
4870 if _object_lock_guard.is_some() {
4871 notify_delete_namespace_acquired(bucket);
4872 }
4873 if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
4874 let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
4875 let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
4876 if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
4877 return Err(StorageError::PreconditionFailed);
4878 }
4879 let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
4880 let (mut current, _) = self
4881 .get_latest_object_info_with_idx(
4882 bucket,
4883 object,
4884 &ObjectOptions {
4885 no_lock: true,
4886 metadata_cache_safe: false,
4887 ..Default::default()
4888 },
4889 )
4890 .await?;
4891 let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
4892 if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
4893 return Err(StorageError::PreconditionFailed);
4894 }
4895 current.name = decode_dir_object(¤t.name);
4896 let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), ¤t).await;
4897 if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
4898 return Err(StorageError::PreconditionFailed);
4899 }
4900 }
4901 if opts.delete_prefix {
4902 delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4903 return Ok(ObjectInfo::default());
4904 }
4905
4906 let creates_latest_marker = latest_versioned_delete_creates_distinct_marker(&opts)
4907 || (opts.version_suspended && should_create_delete_marker_for_missing_object(&opts));
4908 let mut gopts = delete_pool_lookup_opts(&opts, true);
4909 if creates_latest_marker {
4910 gopts.skip_rebalancing = false;
4913 }
4914
4915 if opts.data_movement {
4916 let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
4917 let existing_pool_idx = existing_pool_info
4918 .as_ref()
4919 .map(|(pinfo, _)| pinfo.index)
4920 .map_err(Clone::clone);
4921 let selected_target_pool_idx =
4922 match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
4923 Some(pool_idx) => pool_idx,
4924 None => {
4925 if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4926 self.select_decommission_capacity_target_pool(owner, 0).await?
4927 } else {
4928 self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4929 .await
4930 .ok_or(Error::DiskFull)?
4931 }
4932 }
4933 };
4934 let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
4935 if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4936 Some(self.select_decommission_capacity_target_pool(owner, 0).await?)
4937 } else {
4938 self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4939 .await
4940 }
4941 } else {
4942 None
4943 };
4944 let target_pool_idx =
4945 resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
4946 let mut delete_marker_target_opts = None;
4947
4948 if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4949 let source = self
4950 .find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
4951 .await?;
4952 let Some(source) = source else {
4953 return Err(StorageError::DataMovementOverwriteErr(
4954 bucket.to_owned(),
4955 object.to_owned(),
4956 opts.version_id.unwrap_or_default(),
4957 ));
4958 };
4959 if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
4960 return Err(StorageError::DataMovementOverwriteErr(
4961 bucket.to_owned(),
4962 object.to_owned(),
4963 opts.version_id.unwrap_or_default(),
4964 ));
4965 }
4966 let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
4967 return Err(StorageError::DataMovementOverwriteErr(
4968 bucket.to_owned(),
4969 object.to_owned(),
4970 opts.version_id.unwrap_or_default(),
4971 ));
4972 };
4973 let target = self
4974 .find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
4975 .await?;
4976 if let Some(target) = target {
4977 if is_equivalent_data_movement_delete_marker(&source, &target) {
4978 if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) {
4979 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
4980 .await?;
4981 }
4982 let mut target = target;
4983 target.name = decode_dir_object(object);
4984 return Ok(target);
4985 }
4986 return Err(StorageError::DataMovementOverwriteErr(
4987 bucket.to_owned(),
4988 object.to_owned(),
4989 opts.version_id.unwrap_or_default(),
4990 ));
4991 }
4992 delete_marker_target_opts = Some(target_opts);
4993 }
4994
4995 if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4996 if let Ok((source_pool_info, _)) = existing_pool_info
4997 && opts.delete_marker
4998 && is_data_movement_delete_marker(&source_pool_info.object_info)
4999 && self
5000 .has_equivalent_data_movement_delete_marker(
5001 bucket,
5002 object,
5003 &source_pool_info.object_info,
5004 &opts,
5005 target_pool_idx,
5006 )
5007 .await?
5008 {
5009 let mut obj = source_pool_info.object_info;
5010 obj.name = decode_dir_object(object);
5011 return Ok(obj);
5012 }
5013
5014 return Err(StorageError::DataMovementOverwriteErr(
5015 bucket.to_owned(),
5016 object.to_owned(),
5017 opts.version_id.unwrap_or_default(),
5018 ));
5019 }
5020
5021 let target_opts = delete_marker_target_opts.unwrap_or(opts);
5022 let mut obj = self
5023 .run_decommission_capacity_admitted_mutation_with_capacity_lease(
5024 target_pool_idx,
5025 DecommissionCapacityOwner::from_options(&target_opts),
5026 None,
5027 |capacity_lease| async move {
5028 let mut target_opts = target_opts;
5029 if let Some(capacity_lease) = capacity_lease {
5030 target_opts.add_namespace_lock_lost_signal(capacity_lease);
5031 }
5032 self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await
5033 },
5034 )
5035 .await?;
5036 obj.name = decode_dir_object(obj.name.as_str());
5037 return Ok(obj);
5038 }
5039
5040 let existing_pool_info = if creates_latest_marker {
5042 self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
5043 } else {
5044 self.get_pool_info_existing_with_opts(bucket, object, &gopts).await
5045 };
5046 let (mut pinfo, errs) = match existing_pool_info {
5047 Ok(res) => res,
5048 Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
5049 Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
5050 let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5051 let pool = self.pools[target_pool_idx].clone();
5052 let mut obj = self
5053 .run_external_decommission_capacity_object_mutation(
5054 target_pool_idx,
5055 bucket,
5056 object,
5057 object,
5058 opts,
5059 |opts| async move { pool.delete_object(bucket, object, opts).await },
5060 )
5061 .await?;
5062 #[cfg(test)]
5063 pause_versioned_delete_marker_after_commit(bucket, object).await;
5064 obj.name = decode_dir_object(object);
5065 return Ok(obj);
5066 }
5067 Err(err) => {
5068 if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
5073 return Ok(ObjectInfo {
5074 bucket: bucket.to_owned(),
5075 name: decode_dir_object(object),
5076 ..Default::default()
5077 });
5078 }
5079 return Err(err);
5080 }
5081 };
5082
5083 if creates_latest_marker && self.is_suspended(pinfo.index).await {
5084 let has_active_reservation = self
5085 .pool_meta
5086 .read()
5087 .await
5088 .has_active_decommission_capacity_reservation(pinfo.index);
5089 if has_active_reservation {
5090 pinfo.index = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5091 }
5092 }
5093
5094 if pinfo.object_info.delete_marker && opts.version_id.is_none() && !creates_latest_marker {
5095 pinfo.object_info.name = decode_dir_object(object);
5096 return Ok(pinfo.object_info);
5097 }
5098
5099 if opts.data_movement && opts.src_pool_idx == pinfo.index {
5100 return Err(StorageError::DataMovementOverwriteErr(
5101 bucket.to_owned(),
5102 object.to_owned(),
5103 opts.version_id.unwrap_or_default(),
5104 ));
5105 }
5106
5107 if should_delete_from_all_pools(&opts, errs.len()) {
5108 let mut obj = self
5109 .delete_object_from_all_pools(bucket, object, &opts, &pinfo.object_info, errs)
5110 .await?;
5111 obj.name = decode_dir_object(object);
5112 return Ok(obj);
5113 }
5114
5115 for pool in self.pools.iter() {
5116 if creates_latest_marker && pool.pool_idx != pinfo.index {
5117 continue;
5118 }
5119 if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
5120 if creates_latest_marker {
5121 return Err(StorageError::SlowDown);
5122 }
5123 continue;
5124 }
5125
5126 let pool_idx = pool.pool_idx;
5127 let pool = pool.clone();
5128 match self
5129 .run_external_decommission_capacity_object_delete(
5130 pool_idx,
5131 bucket,
5132 object,
5133 object,
5134 opts.clone(),
5135 |opts| async move { pool.delete_object(bucket, object, opts).await },
5136 )
5137 .await
5138 {
5139 Ok(res) => {
5140 #[cfg(test)]
5141 pause_versioned_delete_marker_after_commit(bucket, object).await;
5142 let mut obj = res;
5143 obj.name = decode_dir_object(object);
5144 return Ok(obj);
5145 }
5146 Err(err) => {
5147 if creates_latest_marker || (!is_err_object_not_found(&err) && !is_err_version_not_found(&err)) {
5148 return Err(err);
5149 }
5150 }
5151 }
5152 }
5153
5154 if let Some(ver) = opts.version_id {
5155 return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
5156 }
5157
5158 Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
5159 }
5160
5161 #[instrument(skip(self, objects, opts))]
5162 pub(super) async fn handle_delete_objects(
5163 &self,
5164 bucket: &str,
5165 objects: Vec<ObjectToDelete>,
5166 opts: ObjectOptions,
5167 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5168 self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
5169 }
5170
5171 pub(super) async fn handle_delete_objects_with_journal(
5172 &self,
5173 bucket: &str,
5174 objects: Vec<ObjectToDelete>,
5175 opts: ObjectOptions,
5176 tier_journal_api: Option<Arc<ECStore>>,
5177 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5178 let (deleted, errors, _) = self
5179 .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
5180 .await;
5181 (deleted, errors)
5182 }
5183
5184 pub(super) async fn handle_delete_objects_with_journal_and_accounting(
5185 &self,
5186 bucket: &str,
5187 objects: Vec<ObjectToDelete>,
5188 opts: ObjectOptions,
5189 tier_journal_api: Option<Arc<ECStore>>,
5190 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5191 Box::pin(async move {
5192 let mut opts = opts;
5193 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
5194 let result = self
5195 .handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
5196 .await;
5197 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
5198 result
5199 })
5200 .await
5201 }
5202
5203 async fn handle_delete_objects_with_journal_and_accounting_inner(
5204 &self,
5205 bucket: &str,
5206 objects: Vec<ObjectToDelete>,
5207 opts: ObjectOptions,
5208 tier_journal_api: Option<Arc<ECStore>>,
5209 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5210 let objects: Vec<ObjectToDelete> = objects
5212 .iter()
5213 .map(|v| {
5214 let mut v = v.clone();
5215 v.object_name = encode_dir_object(v.object_name.as_str());
5216 v
5217 })
5218 .collect();
5219
5220 let mut del_objects = vec![DeletedObject::default(); objects.len()];
5222 let mut accounting = vec![None; objects.len()];
5223
5224 let mut del_errs = Vec::with_capacity(objects.len());
5225 for _ in 0..objects.len() {
5226 del_errs.push(None)
5227 }
5228
5229 let mut opts = opts;
5230 opts.tier_delete_journal_api = tier_journal_api;
5231 let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5232 None
5233 } else {
5234 match self.acquire_bucket_lifecycle_read_lock(bucket).await {
5235 Ok(guard) => Some(guard),
5236 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5237 }
5238 };
5239 if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
5240 opts.add_bucket_lifecycle_lock_guard(guard);
5241 }
5242 if opts.delete_replication_config_snapshot.is_none() {
5243 match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
5244 Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
5245 Err(err) => {
5246 let message = err.to_string();
5247 let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
5248 return (del_objects, errors, accounting);
5249 }
5250 }
5251 }
5252 if !is_meta_bucketname(bucket)
5253 && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
5254 {
5255 return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
5256 }
5257 let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
5258 None
5259 } else {
5260 Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
5261 Ok(guard) => guard,
5262 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5263 })
5264 };
5265 if let Some(guard) = _object_lock_metadata_guard.as_ref() {
5266 opts.add_namespace_lock_guard(guard);
5267 }
5268 let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
5269 let (state, incarnation_id, config_revision) =
5270 match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
5271 Ok(snapshot) => snapshot,
5272 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5273 };
5274 opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
5275 self.id,
5276 bucket,
5277 incarnation_id,
5278 config_revision,
5279 state,
5280 )));
5281 Some(incarnation_id)
5282 } else {
5283 None
5284 };
5285 if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
5286 && expected != current
5287 {
5288 return return_batch_delete_lock_error_with_accounting(
5289 objects.as_slice(),
5290 StorageError::BucketNotFound(bucket.to_string()),
5291 );
5292 }
5293 if opts.expected_bucket_incarnation_id.is_none() {
5294 opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
5295 }
5296 #[cfg(any(test, feature = "test-util"))]
5297 if current_bucket_incarnation_id.is_some() {
5298 pause_delete_after_object_lock_snapshot(bucket).await;
5299 }
5300 let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
5301 Ok(guards) => guards,
5302 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5303 };
5304 #[cfg(any(test, feature = "test-util"))]
5305 if !_object_lock_guards.is_empty() {
5306 notify_delete_namespace_acquired(bucket);
5307 }
5308
5309 let delete_config_snapshot = opts
5310 .delete_replication_config_snapshot
5311 .as_deref()
5312 .expect("batch delete replication config snapshot should be loaded");
5313 let latest_marker_objects = objects
5314 .iter()
5315 .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot))
5316 .collect::<Vec<_>>();
5317 let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map(
5318 |(object, creates_marker)| async move {
5319 if *creates_marker {
5320 Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await)
5321 } else {
5322 None
5323 }
5324 },
5325 ))
5326 .await;
5327 let mut marker_target_pool_indices = Vec::with_capacity(objects.len());
5328 for (idx, target_result) in marker_target_results.into_iter().enumerate() {
5329 match target_result {
5330 Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)),
5331 Some(Err(err)) => {
5332 del_errs[idx] = Some(err);
5333 marker_target_pool_indices.push(None);
5334 }
5335 None => marker_target_pool_indices.push(None),
5336 }
5337 }
5338
5339 let _capacity_fence = if !self.single_pool() && latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
5340 let mut target_pool_indices = marker_target_pool_indices.iter().flatten().copied().collect::<Vec<_>>();
5344 target_pool_indices.sort_unstable();
5345 target_pool_indices.dedup();
5346 match self
5347 .acquire_external_decommission_capacity_fence(&target_pool_indices, DecommissionCapacityAdmission::BatchDelete)
5348 .await
5349 {
5350 Ok(fence) => Some(fence),
5351 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5352 }
5353 } else {
5354 None
5355 };
5356
5357 let mut futures = Vec::with_capacity(self.pools.len());
5358 for pool in self.pools.iter() {
5359 if self.is_pool_rebalancing(pool.pool_idx).await {
5360 continue;
5361 }
5362
5363 let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects
5364 .iter()
5365 .enumerate()
5366 .filter(|(idx, _)| {
5367 batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx)
5368 })
5369 .map(|(idx, object)| (idx, object.clone()))
5370 .unzip();
5371 if pool_objects.is_empty() {
5372 continue;
5373 }
5374
5375 let pool_opts = opts.clone();
5376 futures.push(async move {
5377 #[cfg(test)]
5378 let pool_object_names = pool_objects
5379 .iter()
5380 .map(|object| object.object_name.clone())
5381 .collect::<Vec<_>>();
5382 let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
5383 #[cfg(test)]
5384 let result = {
5385 let mut result = result;
5386 let (deleted, errors, _) = &mut result;
5387 inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
5388 result
5389 };
5390 (object_indices, result)
5391 });
5392 }
5393
5394 let results = join_all(futures).await;
5395
5396 for idx in 0..del_objects.len() {
5397 let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
5398 let pool_object_idx = object_indices.binary_search(&idx).ok()?;
5399 Some((&dels[pool_object_idx], &errs[pool_object_idx]))
5400 });
5401 let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results);
5402 if let Some(deleted) = deleted {
5403 del_objects[idx] = deleted;
5404 }
5405 del_errs[idx] = error;
5406
5407 if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] {
5408 del_objects[idx] = DeletedObject {
5409 object_name: objects[idx].object_name.clone(),
5410 version_id: objects[idx].version_id,
5411 ..Default::default()
5412 };
5413 del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone()));
5414 }
5415 }
5416
5417 for (object_indices, (_, _, pool_accounting)) in &results {
5418 for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
5419 accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
5420 }
5421 }
5422
5423 #[cfg(test)]
5424 for (idx, object) in objects.iter().enumerate() {
5425 if del_errs[idx].is_none() && del_objects[idx].delete_marker {
5426 pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await;
5427 }
5428 }
5429
5430 del_objects.iter_mut().for_each(|v| {
5431 v.object_name = decode_dir_object(&v.object_name);
5432 });
5433
5434 (del_objects, del_errs, accounting)
5435
5436 }
5562
5563 #[instrument(skip(self))]
5564 pub(super) async fn handle_add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
5565 let object = encode_dir_object(object);
5566
5567 if self.single_pool() {
5568 let _ = self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
5569 return Ok(());
5570 }
5571
5572 let opts = ObjectOptions {
5573 version_id: Some(version_id.to_string()),
5574 ..Default::default()
5575 };
5576 let (_, idx) = self
5577 .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5578 .await?;
5579
5580 let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
5581 Ok(())
5582 }
5583
5584 #[instrument(skip(self))]
5585 pub(super) async fn handle_transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5586 let object = encode_dir_object(object);
5587 if self.single_pool() {
5588 return self.pools[0].transition_object(bucket, &object, opts).await;
5589 }
5590
5591 let opts = transition_restore_pool_opts(opts);
5592 let (_, idx) = self
5593 .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
5594 .await?;
5595
5596 self.pools[idx].transition_object(bucket, &object, &opts).await
5597 }
5598
5599 #[instrument(skip(self))]
5600 pub(super) async fn handle_restore_transitioned_object(
5601 self: Arc<Self>,
5602 bucket: &str,
5603 object: &str,
5604 opts: &ObjectOptions,
5605 ) -> Result<()> {
5606 let object = encode_dir_object(object);
5607 let mut opts = transition_restore_pool_opts(opts);
5608 if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
5609 opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
5610 }
5611 let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5612 None
5613 } else {
5614 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
5615 };
5616 if let Some(guard) = bucket_lifecycle_guard.as_ref() {
5617 opts.add_bucket_lifecycle_lock_guard(guard);
5618 }
5619 if !is_meta_bucketname(bucket) {
5620 let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
5621 if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
5622 return Err(StorageError::BucketNotFound(bucket.to_string()));
5623 }
5624 }
5625 if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
5626 let expected_incarnation_id = opts
5627 .expected_bucket_incarnation_id
5628 .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?;
5629 let lifecycle_fence = opts
5630 .bucket_lifecycle_lock_fence
5631 .as_ref()
5632 .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?;
5633 let snapshot = match opts.object_lock_config_snapshot.as_ref() {
5634 Some(snapshot) => Arc::clone(snapshot),
5635 None => {
5636 self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
5637 .await?
5638 }
5639 };
5640 if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
5641 return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation"));
5642 }
5643 snapshot.add_lock_fences(&mut opts);
5644 opts.object_lock_config_snapshot = Some(snapshot);
5645 }
5646 let object_name = object.as_str();
5661 if self.single_pool() {
5662 return self.pools[0]
5663 .clone()
5664 .restore_transitioned_object(bucket, object_name, &opts)
5665 .await;
5666 }
5667
5668 let (_, idx) = self
5669 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock))
5670 .await?;
5671
5672 opts.decommission_capacity_admission = Some(Arc::clone(&self));
5673 self.pools[idx]
5674 .clone()
5675 .restore_transitioned_object(bucket, object_name, &opts)
5676 .await
5677 }
5678
5679 #[instrument(skip(self))]
5680 pub(super) async fn handle_put_object_metadata(
5681 &self,
5682 bucket: &str,
5683 object: &str,
5684 opts: &ObjectOptions,
5685 ) -> Result<ObjectInfo> {
5686 let object = encode_dir_object(object);
5687 let mut opts = opts.clone();
5688 opts.metadata_chg = true;
5689 let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5690 None
5691 } else {
5692 let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
5693 let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
5694 if opts
5695 .expected_bucket_incarnation_id
5696 .is_some_and(|expected| expected != current_incarnation_id)
5697 {
5698 return Err(StorageError::BucketNotFound(bucket.to_string()));
5699 }
5700 opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
5701 opts.add_bucket_lifecycle_lock_guard(&guard);
5702 if guard.is_lock_lost() {
5703 return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
5704 }
5705 Some(guard)
5706 };
5707
5708 if self.single_pool() {
5709 let pool = self.pools[0].clone();
5710 let capacity_object = object.clone();
5711 return self
5712 .run_external_decommission_capacity_object_mutation(
5713 0,
5714 bucket,
5715 capacity_object.as_str(),
5716 capacity_object.as_str(),
5717 opts,
5718 |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5719 )
5720 .await;
5721 }
5722
5723 let (_, idx) = self
5724 .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5725 .await?;
5726
5727 let pool = self.pools[idx].clone();
5728 let capacity_object = object.clone();
5729 let result = self
5730 .run_external_decommission_capacity_object_mutation(
5731 idx,
5732 bucket,
5733 capacity_object.as_str(),
5734 capacity_object.as_str(),
5735 opts,
5736 |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5737 )
5738 .await;
5739 drop(bucket_lifecycle_guard);
5740 result
5741 }
5742
5743 #[instrument(skip(self))]
5744 pub(super) async fn handle_get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
5745 let object = encode_dir_object(object);
5746
5747 if self.single_pool() {
5748 return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
5749 }
5750
5751 let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
5752 Ok((*oi.user_tags).clone())
5753 }
5754
5755 #[instrument(level = "debug", skip(self))]
5756 pub(super) async fn handle_put_object_tags(
5757 &self,
5758 bucket: &str,
5759 object: &str,
5760 tags: &str,
5761 opts: &ObjectOptions,
5762 ) -> Result<ObjectInfo> {
5763 let object = encode_dir_object(object);
5764 let object_name = object.as_str();
5765
5766 if self.single_pool() {
5767 return self
5768 .run_external_decommission_capacity_object_mutation(
5769 0,
5770 bucket,
5771 object_name,
5772 object_name,
5773 opts.clone(),
5774 |opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await },
5775 )
5776 .await;
5777 }
5778
5779 let (_, idx) = self
5780 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5781 .await?;
5782
5783 self.run_external_decommission_capacity_object_mutation(
5784 idx,
5785 bucket,
5786 object_name,
5787 object_name,
5788 opts.clone(),
5789 |opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await },
5790 )
5791 .await
5792 }
5793
5794 #[instrument(skip(self))]
5795 pub(super) async fn handle_delete_object_version(
5796 &self,
5797 bucket: &str,
5798 object: &str,
5799 fi: &FileInfo,
5800 force_del_marker: bool,
5801 ) -> Result<()> {
5802 check_del_obj_args(bucket, object)?;
5803
5804 let object = encode_dir_object(object);
5805
5806 if self.single_pool() {
5807 return self.pools[0]
5808 .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
5809 .await;
5810 }
5811 Err(StorageError::NotImplemented)
5812 }
5813
5814 #[instrument(skip(self))]
5815 pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
5816 let object = encode_dir_object(object);
5817 let object_name = object.as_str();
5818
5819 if self.single_pool() {
5820 return self
5821 .run_external_decommission_capacity_object_mutation(
5822 0,
5823 bucket,
5824 object_name,
5825 object_name,
5826 opts.clone(),
5827 |opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await },
5828 )
5829 .await;
5830 }
5831
5832 let (_, idx) = self
5833 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5834 .await?;
5835
5836 self.run_external_decommission_capacity_object_mutation(
5837 idx,
5838 bucket,
5839 object_name,
5840 object_name,
5841 opts.clone(),
5842 |opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await },
5843 )
5844 .await
5845 }
5846
5847 pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5848 let get_object_reader = <Self as crate::storage_api_contracts::object::ObjectIO>::get_object_reader(
5849 self,
5850 bucket,
5851 object,
5852 None,
5853 HeaderMap::new(),
5854 opts,
5855 )
5856 .await?;
5857 let mut reader = get_object_reader.stream;
5859 tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
5860 Ok(())
5861 }
5862}
5863
5864#[cfg(test)]
5865mod tests {
5866 use super::*;
5867 use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
5868 use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
5869 use crate::bucket::metadata_sys::ObjectLockConfigState;
5870 use crate::bucket::replication::{
5871 ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
5872 version_purge_statuses_map,
5873 };
5874 use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
5875 use crate::core::sets::make_local_two_set_sets_with_ctx;
5876 use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx};
5877 use crate::layout::{
5878 endpoints::{Endpoints, PoolEndpoints, SetupType},
5879 format::FormatV3,
5880 };
5881 use crate::object_api::{
5882 GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
5883 lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
5884 };
5885 use crate::set_disk::same_distributed_lock_domain;
5886 use crate::set_disk::{SetDisks, disk_call_counters};
5887 use crate::storage_api_contracts::bucket::MakeBucketOptions;
5888 use crate::storage_api_contracts::lifecycle::TransitionedObject;
5889 use bytes::Bytes;
5890 use std::future::Future as _;
5891 use std::io::Cursor;
5892 use std::sync::Arc;
5893 use std::sync::atomic::{AtomicUsize, Ordering};
5894 use tokio::io::AsyncReadExt;
5895
5896 #[test]
5897 fn predecessor_replay_scope_distinguishes_exact_object_from_recursive_prefix() {
5898 let source = TierDeleteSourceIdentity {
5899 bucket: "bucket".to_string(),
5900 object: "directory/child".to_string(),
5901 version_id: Some("version".to_string()),
5902 versioned: true,
5903 version_suspended: false,
5904 data_dir: Some("data-dir".to_string()),
5905 etag: Some("etag".to_string()),
5906 mod_time: Some("mod-time".to_string()),
5907 };
5908 let persisted_prefix = encode_dir_object("directory/");
5909
5910 assert!(tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, false,));
5911 assert!(!tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, true,));
5912 let mut exact_source = source;
5913 exact_source.object = "directory/".to_string();
5914 assert!(tier_delete_source_matches_replay_scope(&exact_source, "bucket", &persisted_prefix, true,));
5915 }
5916
5917 struct WaitForLockLossReader {
5918 inner: Cursor<Vec<u8>>,
5919 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5920 resume: ParkingMutex<std::sync::mpsc::Receiver<()>>,
5921 }
5922
5923 impl AsyncRead for WaitForLockLossReader {
5924 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5925 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
5926 if let Some(poll_started) = self.poll_started.take() {
5927 let _ = poll_started.send(());
5928 if self.resume.lock().recv_timeout(Duration::from_secs(10)).is_err() {
5929 return Poll::Ready(Err(std::io::Error::new(
5930 std::io::ErrorKind::TimedOut,
5931 "lock-loss signal was not observed during read",
5932 )));
5933 }
5934 }
5935 poll
5936 }
5937 }
5938
5939 struct PermanentlyPendingReader {
5940 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5941 }
5942
5943 impl AsyncRead for PermanentlyPendingReader {
5944 fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5945 if let Some(poll_started) = self.poll_started.take() {
5946 let _ = poll_started.send(());
5947 }
5948 Poll::Pending
5949 }
5950 }
5951
5952 struct CommitLateBodyReader {
5953 inner: Cursor<Vec<u8>>,
5954 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5955 release: Option<tokio::sync::oneshot::Receiver<()>>,
5956 }
5957
5958 impl AsyncRead for CommitLateBodyReader {
5959 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5960 if let Some(poll_started) = self.poll_started.take() {
5961 let _ = poll_started.send(());
5962 }
5963 if let Some(release) = self.release.as_mut() {
5964 match Pin::new(release).poll(cx) {
5965 Poll::Pending => return Poll::Pending,
5966 Poll::Ready(Ok(())) => self.release = None,
5967 Poll::Ready(Err(_)) => {
5968 return Poll::Ready(Err(std::io::Error::new(
5969 std::io::ErrorKind::BrokenPipe,
5970 "commit-late body release sender was dropped",
5971 )));
5972 }
5973 }
5974 }
5975 Pin::new(&mut self.inner).poll_read(cx, buf)
5976 }
5977 }
5978
5979 struct CountingMissHook {
5980 calls: AtomicUsize,
5981 }
5982
5983 async fn refresh_failure_test_guard(
5984 owner: &'static str,
5985 ) -> (
5986 ObjectLockDiagGuard,
5987 Arc<rustfs_lock::distributed_lock::LockLostSignal>,
5988 Arc<RefreshLossLockClient>,
5989 ) {
5990 let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
5991 rustfs_lock::FastObjectLockManager::new(),
5992 )));
5993 let client = Arc::new(RefreshLossLockClient::with_manager(manager));
5994 let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
5995 owner.to_string(),
5996 vec![Arc::clone(&client) as Arc<dyn rustfs_lock::LockClient>],
5997 1,
5998 );
5999 let request =
6000 rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Shared, owner)
6001 .with_ttl(Duration::from_secs(2))
6002 .with_refresh_interval(Duration::from_millis(20));
6003 let guard = namespace_lock
6004 .acquire_guard(&request)
6005 .await
6006 .expect("distributed read lock acquisition should succeed")
6007 .expect("distributed read lock should reach quorum");
6008 let guard = ObjectLockDiagGuard::new(
6009 guard,
6010 false,
6011 "SelectObjectContent",
6012 Some("bucket".to_string()),
6013 Some("object".to_string()),
6014 Some(owner.to_string()),
6015 ObjectLockDiagMode::Read,
6016 );
6017 let signal = guard
6018 .lock_lost_signal()
6019 .expect("a distributed guard should expose a lock-loss signal");
6020 (guard, signal, client)
6021 }
6022
6023 async fn refresh_failure_test_lease(
6024 owner: &'static str,
6025 ) -> (
6026 Arc<SelectObjectSnapshotLease>,
6027 Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6028 Arc<RefreshLossLockClient>,
6029 ) {
6030 let (guard, signal, client) = refresh_failure_test_guard(owner).await;
6031 (Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client)
6032 }
6033
6034 #[async_trait::async_trait]
6035 impl GetObjectBodyCacheHook for CountingMissHook {
6036 async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
6037 self.calls.fetch_add(1, Ordering::Relaxed);
6038 None
6039 }
6040 }
6041
6042 struct BodyCacheHookGuard;
6043
6044 #[test]
6045 fn select_snapshot_deduplicates_same_distributed_lock_domain() {
6046 let first: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6047 let second: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6048 let other: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6049
6050 assert!(same_distributed_lock_domain(
6051 &[Arc::clone(&first), Arc::clone(&second)],
6052 &[Arc::clone(&second), Arc::clone(&first)]
6053 ));
6054 assert!(same_distributed_lock_domain(
6055 &[Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)],
6056 &[Arc::clone(&second), Arc::clone(&first)]
6057 ));
6058 assert!(!same_distributed_lock_domain(&[first, second], &[other]));
6059 }
6060
6061 #[tokio::test]
6062 async fn publication_commit_locks_dedupe_dist_sets_with_same_clients() {
6063 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6064 let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6065 let mut second_set = (*original_sets.disk_set[1]).clone();
6066 second_set.lockers = original_sets.disk_set[0].lockers.clone();
6067 let mut sets = (*original_sets).clone();
6068 sets.disk_set[1] = Arc::new(second_set);
6069 let sets = Arc::new(sets);
6070 ctx.update_erasure_type(SetupType::DistErasure).await;
6071
6072 assert!(
6073 sets.disk_set[0]
6074 .lockers
6075 .iter()
6076 .zip(&sets.disk_set[1].lockers)
6077 .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)),
6078 "the regression requires identical distributed lock clients"
6079 );
6080 assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index);
6081
6082 let pool_config = sets.endpoints.clone();
6083 let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6084 let object = (0..1_000)
6085 .map(|index| format!("decommission-dist-domain-{index}.bin"))
6086 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6087 .expect("a key should hash to the second set namespace");
6088 let publication_guards = store
6089 .acquire_data_movement_publication_write_locks("bucket", &encode_dir_object(&object), 0, 0, true)
6090 .await
6091 .expect("the publication commit lock set should be acquired");
6092 let target_lock = sets.disk_set[1]
6093 .new_ns_lock("bucket", &object)
6094 .await
6095 .expect("the hashed-set namespace lock should be created");
6096 let target_err = target_lock
6097 .get_write_lock(Duration::from_millis(50))
6098 .await
6099 .expect_err("the fixed write fence must conflict through the shared clients");
6100 assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. }));
6101
6102 drop(publication_guards);
6103
6104 let cleanup_object = (0..1_000)
6105 .map(|index| format!("decommission-dist-cleanup-{index}.bin"))
6106 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6107 .expect("a cleanup key should hash to the second set namespace");
6108 let source_fence = store
6109 .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref())
6110 .await
6111 .expect("the fixed distributed cleanup fence should be acquired");
6112 assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence");
6113 let source_lock = sets.disk_set[1]
6114 .new_ns_lock("bucket", &cleanup_object)
6115 .await
6116 .expect("the source-set namespace lock should be created");
6117 let source_err = source_lock
6118 .get_read_lock(Duration::from_millis(50))
6119 .await
6120 .expect_err("the fixed write fence must conflict through the shared clients");
6121 assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. }));
6122 }
6123
6124 #[tokio::test]
6125 async fn physical_absence_proof_blocks_decommission_remote_tuple_publication() {
6126 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6127 let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6128 ctx.update_erasure_type(SetupType::DistErasure).await;
6129 let pool_config = sets.endpoints.clone();
6130 let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6131 let bucket = "remote-tuple-publication-fence";
6132 let object = "transitioned.bin";
6133 let encoded = encode_dir_object(object);
6134 let mut proof_opts = ObjectOptions::default();
6135 let proof = store
6136 .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6137 .await
6138 .expect("the all-physical absence proof should acquire every read domain");
6139
6140 let mut publication = Box::pin(store.acquire_data_movement_publication_write_locks(bucket, &encoded, 0, 0, true));
6145 let blocked = tokio::time::timeout(Duration::from_millis(50), &mut publication).await;
6146 assert!(
6147 blocked.is_err(),
6148 "a decommission publisher must remain blocked while the remote-delete absence proof is held"
6149 );
6150
6151 drop(proof);
6152 publication
6153 .await
6154 .expect("the publisher should acquire its write fence after the proof releases");
6155 }
6156
6157 #[cfg(feature = "test-util")]
6158 #[tokio::test]
6159 async fn restored_transitioned_rebalance_revalidates_source_after_recovery_lock() {
6160 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6161 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6162 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6163 let store = Arc::new(
6164 new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], Arc::clone(&ctx)).await,
6165 );
6166 let bucket = "restored-transitioned-rebalance-publication";
6167 let object = "restored.bin";
6168 for set in [&first_set, &second_set] {
6169 set.make_bucket(bucket, &MakeBucketOptions::default())
6170 .await
6171 .expect("publication-race bucket should be created");
6172 }
6173
6174 let tier_name = "RESTORED-REBALANCE";
6175 crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), tier_name).await;
6176 let lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), tier_name)
6177 .await
6178 .expect("mock tier generation should be available");
6179 let backend_identity = lease.backend_identity();
6180 drop(lease);
6181
6182 let mut metadata = HashMap::new();
6183 for (suffix, value) in [
6184 (rustfs_utils::http::SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.to_string()),
6185 (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/restored.bin".to_string()),
6186 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
6187 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-version".to_string()),
6188 (
6189 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6190 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
6191 ),
6192 (
6193 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6194 rustfs_utils::crypto::hex(backend_identity),
6195 ),
6196 ] {
6197 rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, value);
6198 }
6199 metadata.insert(
6200 "x-amz-restore".to_string(),
6201 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
6202 );
6203 let mut body = PutObjReader::from_vec(b"restored local body".to_vec());
6204 first_set
6205 .put_object(
6206 bucket,
6207 object,
6208 &mut body,
6209 &ObjectOptions {
6210 user_defined: metadata,
6211 no_lock: true,
6212 ..Default::default()
6213 },
6214 )
6215 .await
6216 .expect("restored-transitioned source should be written");
6217 let source = first_set
6218 .get_object_info(
6219 bucket,
6220 object,
6221 &ObjectOptions {
6222 include_part_checksums: true,
6223 no_lock: true,
6224 ..Default::default()
6225 },
6226 )
6227 .await
6228 .expect("restored-transitioned source metadata should be readable");
6229 assert_eq!(source.transitioned_object.status, TRANSITION_COMPLETE);
6230 assert!(source.data_dir.is_some(), "fixture must retain restored local data");
6231
6232 let publisher_first = store
6233 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6234 .await
6235 .expect("rebalance should capture an exact publication capability");
6236 let encoded = encode_dir_object(object);
6237 let publisher_guard = publisher_first
6238 .into_commit_guard(1, bucket, &encoded)
6239 .await
6240 .expect("unchanged source should validate at the publication boundary");
6241 let mut publisher_first_proof_opts = ObjectOptions::default();
6242 let mut publisher_first_proof = Box::pin(store.acquire_all_physical_object_read_locks(
6243 "tier_delete_journal_recovery",
6244 bucket,
6245 &encoded,
6246 &mut publisher_first_proof_opts,
6247 ));
6248 assert!(
6249 tokio::time::timeout(Duration::from_millis(50), &mut publisher_first_proof)
6250 .await
6251 .is_err(),
6252 "recovery must wait while a validated publisher owns the commit scope"
6253 );
6254 drop(publisher_guard);
6255 drop(
6256 publisher_first_proof
6257 .await
6258 .expect("recovery proof should acquire after publisher commit scope releases"),
6259 );
6260
6261 let publication = store
6262 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6263 .await
6264 .expect("recovery-first race should capture the old source identity");
6265
6266 let pre_commit_reader = first_set
6268 .new_ns_lock(bucket, &encoded)
6269 .await
6270 .expect("source read lock should be created")
6271 .get_read_lock(Duration::from_millis(50))
6272 .await
6273 .expect("body staging must not be blocked by an early publication write lock");
6274 drop(pre_commit_reader);
6275
6276 first_set
6277 .delete_object(
6278 bucket,
6279 &encoded,
6280 ObjectOptions {
6281 delete_prefix: true,
6282 delete_prefix_object: true,
6283 data_movement: true,
6284 no_lock: true,
6285 ..Default::default()
6286 },
6287 )
6288 .await
6289 .expect("source cleanup should remove the old physical owner");
6290
6291 let mut proof_opts = ObjectOptions::default();
6292 let proof = store
6293 .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6294 .await
6295 .expect("recovery should hold its all-physical absence proof");
6296 let mut commit = Box::pin(publication.into_commit_guard(1, bucket, &encoded));
6297 assert!(
6298 tokio::time::timeout(Duration::from_millis(50), &mut commit).await.is_err(),
6299 "publisher must wait while recovery owns the physical absence proof"
6300 );
6301 drop(proof);
6302 let err = match commit.await {
6303 Ok(_) => panic!("publisher must reject its stale source snapshot after recovery releases"),
6304 Err(err) => err,
6305 };
6306 assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6307 assert!(
6308 second_set
6309 .get_object_info(bucket, object, &ObjectOptions::default())
6310 .await
6311 .is_err(),
6312 "stale publication must not create a target owner"
6313 );
6314 }
6315
6316 #[tokio::test]
6317 async fn data_movement_put_defers_publication_locks_until_body_complete() {
6318 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6319 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6320 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6321 let store =
6322 Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6323 let bucket = "commit-late-data-movement-put";
6324 let object = "slow-body.bin";
6325 for set in [&first_set, &second_set] {
6326 set.make_bucket(bucket, &MakeBucketOptions::default())
6327 .await
6328 .expect("commit-late PUT bucket should be created");
6329 }
6330
6331 let payload = b"data movement body".to_vec();
6332 let mut source_body = PutObjReader::from_vec(payload.clone());
6333 first_set
6334 .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6335 .await
6336 .expect("data-movement source should be written");
6337 let source = first_set
6338 .get_object_info(bucket, object, &ObjectOptions::default())
6339 .await
6340 .expect("data-movement source should be readable");
6341 let publication = store
6342 .acquire_remote_tuple_publication_fence(bucket, 0, &source, true)
6343 .await
6344 .expect("decommission should capture its publication capability");
6345
6346 let payload_len = i64::try_from(payload.len()).expect("test payload length should fit in i64");
6347 let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6348 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6349 let slow_body = CommitLateBodyReader {
6350 inner: Cursor::new(payload),
6351 poll_started: Some(poll_started_tx),
6352 release: Some(release_rx),
6353 };
6354 let target_set = Arc::clone(&store.pools[1]);
6355 let target_opts = ObjectOptions {
6356 data_movement: true,
6357 src_pool_idx: 0,
6358 http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
6359 mod_time: source.mod_time,
6360 preserve_etag: source.etag.clone(),
6361 user_defined: source.user_defined.as_ref().clone(),
6362 ..Default::default()
6363 };
6364 let worker = tokio::spawn(async move {
6365 let hash_reader = rustfs_rio::HashReader::from_stream(slow_body, payload_len, payload_len, None, None, false)
6366 .expect("slow body should produce a valid hash reader");
6367 let mut reader = PutObjReader::new(hash_reader);
6368 target_set
6369 .put_object_with_old_current_size_for_data_movement(bucket, object, &mut reader, &target_opts, publication)
6370 .await
6371 });
6372
6373 tokio::time::timeout(Duration::from_secs(5), poll_started_rx)
6374 .await
6375 .expect("target PUT should start polling its body")
6376 .expect("target PUT should retain the poll observer");
6377 let source_head = tokio::time::timeout(
6378 Duration::from_millis(250),
6379 first_set.get_object_info(bucket, object, &ObjectOptions::default()),
6380 )
6381 .await
6382 .expect("same-key HEAD must not wait behind body staging")
6383 .expect("the source must remain readable during body staging");
6384 assert_eq!(source_head.etag, source.etag);
6385 let target_head = tokio::time::timeout(
6386 Duration::from_millis(250),
6387 second_set.get_object_info(bucket, object, &ObjectOptions::default()),
6388 )
6389 .await
6390 .expect("target HEAD must not wait behind body staging")
6391 .expect_err("the target must remain unpublished before body EOF");
6392 assert!(is_err_object_not_found(&target_head) || is_err_version_not_found(&target_head));
6393
6394 release_tx
6395 .send(())
6396 .expect("target PUT should still be waiting on the body gate");
6397 worker
6398 .await
6399 .expect("target PUT task should join")
6400 .expect("target PUT should commit after body EOF");
6401 second_set
6402 .get_object_info(bucket, object, &ObjectOptions::default())
6403 .await
6404 .expect("target metadata should be visible after the commit boundary");
6405 }
6406
6407 #[test]
6408 fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() {
6409 let protected_suffixes = [
6410 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6411 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
6412 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
6413 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6414 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
6415 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6416 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
6417 rustfs_utils::http::SUFFIX_FREE_VERSION,
6418 rustfs_utils::http::SUFFIX_TIER_FV_ID,
6419 rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
6420 ];
6421 let mut source = HashMap::from([("content-type".to_string(), "application/octet-stream".to_string())]);
6422 for suffix in protected_suffixes {
6423 rustfs_utils::http::metadata_compat::insert_str(&mut source, suffix, format!("source-{suffix}"));
6424 }
6425 source.insert(
6428 format!(
6429 "{}{}",
6430 rustfs_utils::http::MINIO_INTERNAL_PREFIX.to_ascii_uppercase(),
6431 rustfs_utils::http::SUFFIX_TRANSITION_STATUS.to_ascii_uppercase()
6432 ),
6433 "complete".to_string(),
6434 );
6435
6436 let copied = materialized_copy_user_defined(&source);
6437
6438 assert_eq!(copied.get("content-type").map(String::as_str), Some("application/octet-stream"));
6439 for suffix in protected_suffixes {
6440 assert!(
6441 !rustfs_utils::http::metadata_compat::contains_key_str(&copied, suffix),
6442 "materialized copy retained protected internal suffix {suffix}"
6443 );
6444 }
6445 }
6446
6447 #[test]
6448 fn remote_tuple_publication_target_is_exact_legacy_compatible_and_ordinary_zero_cost() {
6449 let mut ordinary_metadata = HashMap::new();
6450 rustfs_utils::http::metadata_compat::insert_str(
6451 &mut ordinary_metadata,
6452 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6453 "malformed-but-irrelevant".to_string(),
6454 );
6455 let ordinary = ObjectInfo {
6456 user_defined: Arc::new(ordinary_metadata),
6457 ..Default::default()
6458 };
6459 assert!(
6460 remote_tuple_publication_target(&ordinary)
6461 .expect("ordinary objects must not parse or lock tier metadata")
6462 .is_none()
6463 );
6464
6465 let identity = [0xabu8; 32];
6466 let mut exact_metadata = HashMap::new();
6467 rustfs_utils::http::metadata_compat::insert_str(
6468 &mut exact_metadata,
6469 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6470 rustfs_utils::crypto::hex(identity),
6471 );
6472 let transitioned = ObjectInfo {
6473 transitioned_object: TransitionedObject {
6474 status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
6475 name: "remote/object".to_string(),
6476 tier: "WARM".to_string(),
6477 ..Default::default()
6478 },
6479 user_defined: Arc::new(exact_metadata),
6480 ..Default::default()
6481 };
6482 let (tier, recorded_identity) = remote_tuple_publication_target(&transitioned)
6483 .expect("exact transitioned metadata should parse")
6484 .expect("transitioned object should require a publication fence");
6485 assert_eq!(tier, "WARM");
6486 assert_eq!(recorded_identity, Some(identity));
6487
6488 let legacy = ObjectInfo {
6489 user_defined: Arc::new(HashMap::new()),
6490 ..transitioned.clone()
6491 };
6492 assert_eq!(
6493 remote_tuple_publication_target(&legacy)
6494 .expect("legacy transitioned metadata should remain compatible")
6495 .map(|(_, identity)| identity),
6496 Some(None)
6497 );
6498
6499 let mut malformed = transitioned;
6500 malformed.transitioned_object.tier.clear();
6501 assert!(remote_tuple_publication_target(&malformed).is_err());
6502 }
6503
6504 #[test]
6505 fn publication_source_match_rejects_new_physical_generation_with_same_logical_identity() {
6506 let expected = ObjectInfo {
6507 data_dir: Some(Uuid::new_v4()),
6508 ..Default::default()
6509 };
6510 assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6511
6512 let current = ObjectInfo {
6513 data_dir: Some(Uuid::new_v4()),
6514 ..expected.clone()
6515 };
6516 assert!(
6517 crate::data_movement::is_equivalent_data_movement_object_identity(&expected, ¤t, true, true),
6518 "the logical target-equivalence check deliberately ignores target data-dir allocation"
6519 );
6520 assert!(
6521 !remote_tuple_publication_object_source_matches(&expected, ¤t),
6522 "publication revalidation must additionally bind the physical source generation"
6523 );
6524 }
6525
6526 #[test]
6527 fn exact_delete_capacity_target_requires_matching_namespace_and_object_identity() {
6528 let source = ObjectInfo {
6529 bucket: "bucket".to_string(),
6530 name: "directory/".to_string(),
6531 version_id: Some(Uuid::from_u128(1)),
6532 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
6533 size: 10,
6534 etag: Some("source-etag".to_string()),
6535 ..Default::default()
6536 };
6537 let target = ObjectInfo {
6538 name: rustfs_utils::path::encode_dir_object(&source.name),
6539 ..source.clone()
6540 };
6541 assert!(ECStore::is_equivalent_decommission_capacity_target(&source, &target));
6542
6543 let mismatched_identity = ObjectInfo {
6544 etag: Some("different-etag".to_string()),
6545 ..target.clone()
6546 };
6547 assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &mismatched_identity));
6548
6549 let wrong_bucket = ObjectInfo {
6550 bucket: "other-bucket".to_string(),
6551 ..target
6552 };
6553 assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &wrong_bucket));
6554 }
6555
6556 fn publication_part(number: usize) -> ObjectPartInfo {
6557 ObjectPartInfo {
6558 number,
6559 size: 100 + number,
6560 actual_size: i64::try_from(200 + number).expect("test part size should fit in i64"),
6561 etag: format!("part-etag-{number}"),
6562 ..Default::default()
6563 }
6564 }
6565
6566 fn publication_source(parts: Vec<ObjectPartInfo>) -> ObjectInfo {
6567 ObjectInfo {
6568 data_dir: Some(Uuid::new_v4()),
6569 version_id: Some(Uuid::new_v4()),
6570 size: 4096,
6571 actual_size: 4096,
6572 etag: Some("object-etag".to_string()),
6573 checksum: Some(Bytes::from_static(b"object-checksum")),
6574 mod_time: Some(time::OffsetDateTime::UNIX_EPOCH),
6575 parts: Arc::new(parts),
6576 ..Default::default()
6577 }
6578 }
6579
6580 #[test]
6581 fn publication_parts_match_is_order_independent_and_bijective() {
6582 let ordered = vec![publication_part(1), publication_part(2), publication_part(3)];
6583 let mut reversed = ordered.clone();
6584 reversed.reverse();
6585
6586 assert!(remote_tuple_publication_parts_match(&[], &[]));
6587 assert!(remote_tuple_publication_parts_match(&ordered, &ordered));
6588 assert!(remote_tuple_publication_parts_match(&ordered, &reversed));
6589 assert!(!remote_tuple_publication_parts_match(&ordered[..2], &ordered));
6590
6591 let expected_duplicate = vec![publication_part(1), publication_part(1)];
6592 let current_unique = vec![publication_part(1), publication_part(2)];
6593 assert!(
6594 !remote_tuple_publication_parts_match(&expected_duplicate, ¤t_unique),
6595 "two expected entries must not reuse the same current part"
6596 );
6597 assert!(!remote_tuple_publication_parts_match(¤t_unique, &expected_duplicate));
6598 assert!(!remote_tuple_publication_parts_match(&expected_duplicate, &expected_duplicate));
6599 }
6600
6601 #[test]
6602 fn publication_parts_match_preserves_exact_part_identity_contract() {
6603 let expected = vec![publication_part(1)];
6604
6605 let mut different_number = expected.clone();
6606 different_number[0].number = 2;
6607 assert!(!remote_tuple_publication_parts_match(&expected, &different_number));
6608
6609 let mut different_size = expected.clone();
6610 different_size[0].size += 1;
6611 assert!(!remote_tuple_publication_parts_match(&expected, &different_size));
6612
6613 let mut different_actual_size = expected.clone();
6614 different_actual_size[0].actual_size += 1;
6615 assert!(!remote_tuple_publication_parts_match(&expected, &different_actual_size));
6616
6617 let mut different_etag = expected.clone();
6618 different_etag[0].etag.push_str("-changed");
6619 assert!(!remote_tuple_publication_parts_match(&expected, &different_etag));
6620
6621 let mut ignored_fields = expected.clone();
6622 ignored_fields[0].index = Some(Bytes::from_static(b"different-index"));
6623 ignored_fields[0].checksums = Some(HashMap::from([("CRC32C".to_string(), "different".to_string())]));
6624 ignored_fields[0].mod_time = Some(time::OffsetDateTime::UNIX_EPOCH);
6625 assert!(
6626 remote_tuple_publication_parts_match(&expected, &ignored_fields),
6627 "the publication fence must retain its existing checksum/index/mod-time compatibility contract"
6628 );
6629
6630 let zero_actual_size = vec![ObjectPartInfo {
6631 actual_size: 0,
6632 ..publication_part(1)
6633 }];
6634 assert!(
6635 !remote_tuple_publication_parts_match(&zero_actual_size, &expected),
6636 "the publication fence compares raw part actual sizes without the broader comparator's fallback"
6637 );
6638 }
6639
6640 #[test]
6641 fn publication_source_match_rejects_object_etag_and_identity_mutations() {
6642 let expected = publication_source(vec![publication_part(1)]);
6643 assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6644
6645 let mut current = expected.clone();
6646 current.etag = Some("changed-object-etag".to_string());
6647 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6648
6649 let mut current = expected.clone();
6650 current.checksum = Some(Bytes::from_static(b"changed-checksum"));
6651 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6652
6653 let mut current = expected.clone();
6654 current.actual_size += 1;
6655 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6656
6657 let mut current = expected.clone();
6658 current.data_dir = Some(Uuid::new_v4());
6659 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6660
6661 let mut current = expected.clone();
6662 current.version_id = Some(Uuid::new_v4());
6663 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6664
6665 let mut current = expected.clone();
6666 current.mod_time = current.mod_time.map(|mod_time| mod_time + time::Duration::SECOND);
6667 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6668 }
6669
6670 #[test]
6671 fn publication_source_match_preserves_effective_actual_size_compatibility() {
6672 let expected = publication_source(vec![publication_part(1)]);
6673 let current = ObjectInfo {
6674 actual_size: 0,
6675 ..expected.clone()
6676 };
6677
6678 assert_eq!(
6679 expected.get_actual_size().expect("expected size should be valid"),
6680 current.get_actual_size().expect("current size should be valid")
6681 );
6682 assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
6683 }
6684
6685 #[test]
6686 fn publication_source_match_rejects_duplicate_parts_through_the_full_fence() {
6687 let expected = publication_source(vec![publication_part(1), publication_part(1)]);
6688 let current = ObjectInfo {
6689 parts: Arc::new(vec![publication_part(1), publication_part(2)]),
6690 ..expected.clone()
6691 };
6692
6693 assert!(
6694 !remote_tuple_publication_object_source_matches(&expected, ¤t),
6695 "the full source fence must reject the original non-bijective false-positive"
6696 );
6697 }
6698
6699 #[test]
6700 fn publication_source_match_handles_10_000_reversed_parts_without_payload_cloning() {
6701 let parts = Arc::new((1..=10_000).map(publication_part).collect::<Vec<_>>());
6702 let mut reversed = parts.as_ref().clone();
6703 reversed.reverse();
6704 let expected = publication_source(Vec::new());
6705 let expected = ObjectInfo {
6706 parts: Arc::clone(&parts),
6707 ..expected
6708 };
6709 let current = ObjectInfo {
6710 parts: Arc::new(reversed),
6711 ..expected.clone()
6712 };
6713 let expected_parts = Arc::clone(&expected.parts);
6714
6715 assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
6716 assert!(Arc::ptr_eq(&expected.parts, &expected_parts));
6717 }
6718
6719 #[tokio::test]
6720 async fn publication_commit_guard_rejects_an_etag_only_source_change() {
6721 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6722 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6723 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6724 let store =
6725 Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6726 let bucket = "publication-etag-source-change";
6727 let object = "source.bin";
6728 for set in [&first_set, &second_set] {
6729 set.make_bucket(bucket, &MakeBucketOptions::default())
6730 .await
6731 .expect("publication ETag test bucket should be created");
6732 }
6733
6734 let mut source_body = PutObjReader::from_vec(b"source body".to_vec());
6735 first_set
6736 .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6737 .await
6738 .expect("publication source should be written");
6739 let source = first_set
6740 .get_object_info(
6741 bucket,
6742 object,
6743 &ObjectOptions {
6744 include_part_checksums: true,
6745 ..Default::default()
6746 },
6747 )
6748 .await
6749 .expect("publication source should be readable");
6750 let publication = store
6751 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6752 .await
6753 .expect("the source snapshot should produce a publication capability");
6754
6755 let changed = first_set
6756 .put_object_metadata(
6757 bucket,
6758 object,
6759 &ObjectOptions {
6760 eval_metadata: Some(HashMap::from([("etag".to_string(), "changed-etag".to_string())])),
6761 ..Default::default()
6762 },
6763 )
6764 .await
6765 .expect("the source ETag should be updated in place");
6766 assert_ne!(source.etag, changed.etag);
6767 let unchanged_except_etag = ObjectInfo {
6768 etag: source.etag.clone(),
6769 ..changed.clone()
6770 };
6771 assert!(
6772 remote_tuple_publication_object_source_matches(&source, &unchanged_except_etag),
6773 "the persisted source update fixture must differ only by object ETag"
6774 );
6775
6776 let encoded = encode_dir_object(object);
6777 let err = match publication.into_commit_guard(1, bucket, &encoded).await {
6778 Ok(_) => panic!("the publication guard must reject an ETag-only source change"),
6779 Err(err) => err,
6780 };
6781 assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6782 assert!(
6783 second_set
6784 .get_object_info(bucket, object, &ObjectOptions::default())
6785 .await
6786 .is_err(),
6787 "a rejected publication must not create a target object"
6788 );
6789 }
6790
6791 #[tokio::test]
6792 async fn generic_data_movement_put_rejects_transition_ownership_without_capability() {
6793 let (_dirs, set) = make_local_set_disks(4, 2).await;
6794 let bucket = "unfenced-transition-publication";
6795 set.make_bucket(bucket, &MakeBucketOptions::default())
6796 .await
6797 .expect("unfenced publication bucket should be created");
6798 let mut metadata = HashMap::new();
6799 rustfs_utils::http::metadata_compat::insert_str(
6800 &mut metadata,
6801 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6802 TRANSITION_COMPLETE.to_string(),
6803 );
6804 let mut body = PutObjReader::from_vec(b"must not publish".to_vec());
6805 let err = set
6806 .put_object(
6807 bucket,
6808 "unfenced.bin",
6809 &mut body,
6810 &ObjectOptions {
6811 data_movement: true,
6812 user_defined: metadata,
6813 ..Default::default()
6814 },
6815 )
6816 .await
6817 .expect_err("transition ownership must require the typed publication entry point");
6818 assert!(err.to_string().contains("publication capability"));
6819 }
6820
6821 #[test]
6822 fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() {
6823 let nil = Uuid::nil();
6824 for actual in [None, Some(nil)] {
6825 assert!(select_snapshot_version_matches(actual, "null"));
6826 assert!(select_snapshot_version_matches(actual, "NULL"));
6827 assert!(select_snapshot_version_matches(actual, &nil.to_string()));
6828 }
6829
6830 let version = Uuid::new_v4();
6831 assert!(select_snapshot_version_matches(Some(version), &version.to_string().to_uppercase()));
6832 assert!(!select_snapshot_version_matches(Some(version), "null"));
6833 assert!(!select_snapshot_version_matches(Some(version), "not-a-version"));
6834 }
6835
6836 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6837 async fn select_snapshot_reader_rolls_back_bytes_when_lock_is_lost_during_poll() {
6838 let (lease, signal, client) = refresh_failure_test_lease("select-snapshot-lock-loss").await;
6839
6840 let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6841 let (resume_tx, resume_rx) = std::sync::mpsc::channel();
6842 let release_client = Arc::clone(&client);
6843 let release_lease = Arc::clone(&lease);
6844 let release_signal = Arc::clone(&signal);
6845 let release_task = tokio::spawn(async move {
6846 poll_started_rx.await.expect("reader poll should start");
6847 release_client.reject_refreshes();
6848 release_client
6849 .wait_for_rejected_refresh(Duration::from_secs(5))
6850 .await
6851 .expect("refresh rejection should be observed");
6852 tokio::time::timeout(Duration::from_secs(5), release_signal.notified())
6853 .await
6854 .expect("heartbeat should observe the rejected refresh");
6855 tokio::time::timeout(Duration::from_secs(5), async {
6856 while !release_lease.is_lost() {
6857 tokio::task::yield_now().await;
6858 }
6859 })
6860 .await
6861 .expect("snapshot monitor should publish lock loss");
6862 resume_tx.send(()).expect("reader poll should still be waiting");
6863 });
6864
6865 let mut reader = SelectObjectSnapshotReader {
6866 inner: Box::new(WaitForLockLossReader {
6867 inner: Cursor::new(b"must-not-escape".to_vec()),
6868 poll_started: Some(poll_started_tx),
6869 resume: ParkingMutex::new(resume_rx),
6870 }),
6871 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6872 lease,
6873 };
6874 let mut output = Vec::new();
6875 let error = reader
6876 .read_to_end(&mut output)
6877 .await
6878 .expect_err("a read that loses its lease must fail");
6879 release_task.await.expect("backend release task should not panic");
6880
6881 assert!(signal.is_lost(), "heartbeat should report the rejected refresh");
6882 assert!(output.is_empty(), "bytes read during the failed poll must be rolled back");
6883 assert_eq!(error.kind(), std::io::ErrorKind::Other);
6884 }
6885
6886 #[tokio::test(flavor = "current_thread")]
6887 async fn select_snapshot_reader_checks_guards_before_monitor_runs() {
6888 let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await;
6889 client.reject_refreshes();
6890 client
6891 .wait_for_rejected_refresh(Duration::from_secs(5))
6892 .await
6893 .expect("refresh rejection should be observed");
6894 tokio::time::timeout(Duration::from_secs(5), signal.notified())
6895 .await
6896 .expect("heartbeat should observe the rejected refresh");
6897
6898 let lease = Arc::new(SelectObjectSnapshotLease::new(vec![guard]));
6899 assert!(!lease.is_lost(), "current-thread monitor must not run before the synchronous read");
6900 let mut reader = SelectObjectSnapshotReader {
6901 inner: Box::new(Cursor::new(b"old-generation".to_vec())),
6902 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6903 lease,
6904 };
6905 let mut output = Vec::new();
6906
6907 let error = reader
6908 .read_to_end(&mut output)
6909 .await
6910 .expect_err("EOF fence must reject a lease lost before its monitor is scheduled");
6911
6912 assert!(output.is_empty(), "bytes from a known-lost snapshot must not escape");
6913 assert_eq!(error.kind(), std::io::ErrorKind::Other);
6914 }
6915
6916 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6917 async fn select_snapshot_lock_loss_is_broadcast_to_remaining_pending_readers() {
6918 let (first_guard, first_signal, first_client) = refresh_failure_test_guard("select-snapshot-pending-first-lock").await;
6919 let (second_guard, second_signal, second_client) =
6920 refresh_failure_test_guard("select-snapshot-pending-second-lock").await;
6921 let lease = Arc::new(SelectObjectSnapshotLease::new(vec![first_guard, second_guard]));
6922 let dropped_reader = SelectObjectSnapshotReader {
6923 inner: Box::new(PermanentlyPendingReader { poll_started: None }),
6924 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6925 lease: Arc::clone(&lease),
6926 };
6927 drop(dropped_reader);
6928
6929 let (first_started_tx, first_started_rx) = tokio::sync::oneshot::channel();
6930 let (second_started_tx, second_started_rx) = tokio::sync::oneshot::channel();
6931 let mut first_reader = SelectObjectSnapshotReader {
6932 inner: Box::new(PermanentlyPendingReader {
6933 poll_started: Some(first_started_tx),
6934 }),
6935 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6936 lease: Arc::clone(&lease),
6937 };
6938 let mut second_reader = SelectObjectSnapshotReader {
6939 inner: Box::new(PermanentlyPendingReader {
6940 poll_started: Some(second_started_tx),
6941 }),
6942 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6943 lease,
6944 };
6945 let first_read_task = tokio::spawn(async move {
6946 let mut byte = [0_u8; 1];
6947 first_reader.read(&mut byte).await
6948 });
6949 let second_read_task = tokio::spawn(async move {
6950 let mut byte = [0_u8; 1];
6951 second_reader.read(&mut byte).await
6952 });
6953
6954 first_started_rx.await.expect("first inner reader should reach Poll::Pending");
6955 second_started_rx
6956 .await
6957 .expect("second inner reader should reach Poll::Pending");
6958 second_client.reject_refreshes();
6959 second_client
6960 .wait_for_rejected_refresh(Duration::from_secs(5))
6961 .await
6962 .expect("refresh rejection should be observed");
6963 let (first_result, second_result) = tokio::join!(
6964 tokio::time::timeout(Duration::from_secs(5), first_read_task),
6965 tokio::time::timeout(Duration::from_secs(5), second_read_task),
6966 );
6967 for result in [first_result, second_result] {
6968 let error = result
6969 .expect("lock loss should wake every reader whose inner I/O remains pending")
6970 .expect("reader task should not panic")
6971 .expect_err("lost snapshot lease must fail every pending read");
6972 assert_eq!(error.kind(), std::io::ErrorKind::Other);
6973 }
6974
6975 assert!(!first_client.refreshes_rejected());
6976 assert!(!first_signal.is_lost());
6977 assert!(second_signal.is_lost());
6978 }
6979
6980 #[test]
6981 fn tier_delete_entry_is_prepared_and_prefix_legacy_fails_closed() {
6982 let identity = [9_u8; 32];
6983 let mut metadata = HashMap::new();
6984 rustfs_utils::http::metadata_compat::insert_str(
6985 &mut metadata,
6986 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6987 rustfs_utils::crypto::hex(identity),
6988 );
6989 let version_id = Uuid::from_u128(1);
6990 let data_dir = Uuid::from_u128(2);
6991 let source = ObjectInfo {
6992 bucket: "bucket".to_string(),
6993 name: "object".to_string(),
6994 version_id: Some(version_id),
6995 data_dir: Some(data_dir),
6996 user_defined: Arc::new(metadata),
6997 transitioned_object: TransitionedObject {
6998 name: "remote/object".to_string(),
6999 version_id: "remote-version".to_string(),
7000 tier: "WARM".to_string(),
7001 status: TRANSITION_COMPLETE.to_string(),
7002 ..Default::default()
7003 },
7004 transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
7005 ..Default::default()
7006 };
7007 let entry = build_tier_delete_journal_entry(
7008 "bucket",
7009 "object",
7010 &ObjectOptions {
7011 version_id: Some(version_id.to_string()),
7012 versioned: true,
7013 ..Default::default()
7014 },
7015 &source,
7016 )
7017 .expect("transition source should produce a journal entry")
7018 .expect("completed transition should be journaled");
7019
7020 assert_eq!(entry.state, TierDeleteJournalState::Prepared);
7021 assert_eq!(entry.backend_identity, Some(identity));
7022 let data_dir_string = data_dir.to_string();
7023 assert_eq!(
7024 entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
7025 Some(data_dir_string.as_str())
7026 );
7027
7028 let mut unknown = source.clone();
7029 unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7030 assert!(
7031 build_tier_delete_journal_entry(
7032 "bucket",
7033 "object",
7034 &ObjectOptions {
7035 delete_prefix: true,
7036 versioned: true,
7037 ..Default::default()
7038 },
7039 &unknown,
7040 )
7041 .expect_err("recursive prefix delete must fail closed for legacy Unknown metadata")
7042 .to_string()
7043 .contains("legacy Unknown")
7044 );
7045
7046 assert!(
7047 build_tier_delete_journal_entry(
7048 "bucket",
7049 "object",
7050 &ObjectOptions {
7051 delete_prefix: true,
7052 delete_prefix_object: true,
7053 versioned: true,
7054 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
7055 version_id: Some(version_id),
7056 delete_marker: false,
7057 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
7058 rule_id: "delete-all".to_string(),
7059 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
7060 }),
7061 ..Default::default()
7062 },
7063 &unknown,
7064 )
7065 .expect("lifecycle exact-object delete-all may use its free-version fallback")
7066 .is_none()
7067 );
7068
7069 let mut identity_less = source;
7070 identity_less.user_defined = Arc::new(HashMap::new());
7071 assert!(
7072 build_tier_delete_journal_entry(
7073 "bucket",
7074 "object",
7075 &ObjectOptions {
7076 delete_prefix: true,
7077 versioned: true,
7078 ..Default::default()
7079 },
7080 &identity_less,
7081 )
7082 .expect_err("recursive prefix delete must fail closed without destination identity")
7083 .to_string()
7084 .contains("requires a stable transitioned source")
7085 );
7086 }
7087
7088 #[test]
7089 fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
7090 let cancelled = Error::OperationCanceled;
7091 assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
7092 assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
7093 assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
7094 assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
7095 }
7096
7097 #[test]
7098 fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
7099 let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
7100 .expect_err("a real walk error must not be hidden by earlier cancellation");
7101 assert_eq!(err, StorageError::FileAccessDenied);
7102
7103 assert_eq!(
7104 combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
7105 .expect_err("cancellation must remain visible when there is no real error"),
7106 Error::OperationCanceled
7107 );
7108 }
7109
7110 impl Drop for BodyCacheHookGuard {
7111 fn drop(&mut self) {
7112 clear_get_object_body_cache_hook();
7113 }
7114 }
7115
7116 #[test]
7117 fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
7118 let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
7119 assert_eq!(target, None);
7120 }
7121
7122 #[test]
7123 fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
7124 let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
7125 let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
7126 assert_eq!(target, None);
7127 }
7128
7129 #[test]
7130 fn non_delete_marker_data_movement_keeps_existing_pool() {
7131 let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
7132 assert_eq!(target, Some(0));
7133 }
7134
7135 #[test]
7136 fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
7137 let version_id = Uuid::nil();
7138 let mod_time = OffsetDateTime::UNIX_EPOCH;
7139 let source = ObjectInfo {
7140 version_id: Some(version_id),
7141 delete_marker: true,
7142 mod_time: Some(mod_time),
7143 ..Default::default()
7144 };
7145 let target = source.clone();
7146
7147 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7148
7149 let mismatched = ObjectInfo {
7150 mod_time: Some(mod_time + Duration::from_secs(1)),
7151 ..target
7152 };
7153 assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
7154 }
7155
7156 #[test]
7157 fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
7158 let mut source = ObjectInfo {
7159 version_id: Some(Uuid::from_u128(1)),
7160 delete_marker: true,
7161 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7162 ..Default::default()
7163 };
7164 rustfs_utils::http::insert_str(
7165 Arc::make_mut(&mut source.user_defined),
7166 rustfs_utils::http::SUFFIX_TIER_FV_ID,
7167 Uuid::from_u128(2).to_string(),
7168 );
7169 let mut target = source.clone();
7170 rustfs_utils::http::insert_str(
7171 Arc::make_mut(&mut target.user_defined),
7172 rustfs_utils::http::SUFFIX_TIER_FV_ID,
7173 Uuid::from_u128(3).to_string(),
7174 );
7175
7176 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7177
7178 Arc::make_mut(&mut target.user_defined).insert(
7179 format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
7180 Uuid::from_u128(4).to_string(),
7181 );
7182 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7183 }
7184
7185 #[test]
7186 fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
7187 let key = format!(
7188 "{}{}",
7189 rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7190 rustfs_utils::http::SUFFIX_REPLICATION_STATUS
7191 );
7192 let timestamp_key = format!(
7193 "{}{}",
7194 rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7195 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
7196 );
7197 let source = ObjectInfo {
7198 version_id: Some(Uuid::from_u128(1)),
7199 delete_marker: true,
7200 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7201 user_defined: Arc::new(HashMap::from([
7202 (key.clone(), "arn=COMPLETED;".to_string()),
7203 (timestamp_key, "1970-01-01T00:00:01Z".to_string()),
7204 ])),
7205 ..Default::default()
7206 };
7207 let mut target = source.clone();
7208 rustfs_utils::http::insert_str(
7209 Arc::make_mut(&mut target.user_defined),
7210 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7211 "arn=COMPLETED;".to_string(),
7212 );
7213 rustfs_utils::http::insert_str(
7214 Arc::make_mut(&mut target.user_defined),
7215 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
7216 (OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
7217 );
7218 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7219
7220 Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
7221 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7222 }
7223
7224 #[test]
7225 fn equivalent_data_movement_delete_marker_ignores_target_bucket_incarnation_fence() {
7226 let source = ObjectInfo {
7227 version_id: Some(Uuid::from_u128(1)),
7228 delete_marker: true,
7229 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7230 ..Default::default()
7231 };
7232 let mut target = source.clone();
7233 rustfs_utils::http::insert_str(
7234 Arc::make_mut(&mut target.user_defined),
7235 rustfs_utils::http::SUFFIX_BUCKET_INCARNATION_ID,
7236 Uuid::from_u128(2).to_string(),
7237 );
7238
7239 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7240 }
7241
7242 #[test]
7243 fn data_movement_delete_marker_source_requires_persisted_mod_time() {
7244 let source = ObjectInfo {
7245 delete_marker: true,
7246 ..Default::default()
7247 };
7248 assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7249
7250 let source = ObjectInfo {
7251 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7252 ..source
7253 };
7254 assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
7255 assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7256 }
7257
7258 #[test]
7259 fn data_movement_delete_marker_uses_current_source_replication_state() {
7260 let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
7261 let timestamp = expected_timestamp.to_string();
7262 let mut metadata = HashMap::new();
7263 rustfs_utils::http::insert_str(
7264 &mut metadata,
7265 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7266 ReplicationStatusType::Replica.to_string(),
7267 );
7268 rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
7269 rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
7270 rustfs_utils::http::insert_str(
7271 &mut metadata,
7272 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7273 "arn=COMPLETED;".to_string(),
7274 );
7275 rustfs_utils::http::insert_str(
7276 &mut metadata,
7277 &format!(
7278 "{}{}",
7279 rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
7280 "arn:minio:replication::TenantA:bucket"
7281 ),
7282 "reset-id".to_string(),
7283 );
7284 rustfs_utils::http::insert_str(
7285 &mut metadata,
7286 &format!(
7287 "{}{}",
7288 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
7289 "arn:minio:replication::TenantA:bucket"
7290 ),
7291 "target-version".to_string(),
7292 );
7293 let source = ObjectInfo {
7294 delete_marker: true,
7295 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7296 replication_status_internal: Some("arn=COMPLETED;".to_string()),
7297 replication_decision: "arn=replicate;".to_string(),
7298 user_defined: Arc::new(metadata),
7299 ..Default::default()
7300 };
7301 let opts = ObjectOptions {
7302 mod_time: source.mod_time,
7303 delete_replication: Some(ReplicationState {
7304 replication_status_internal: Some("arn=PENDING;".to_string()),
7305 ..Default::default()
7306 }),
7307 ..Default::default()
7308 };
7309
7310 let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
7311 let state = target_opts.delete_replication.as_ref().expect("current replication state");
7312 assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
7313 assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
7314 assert_eq!(state.replica_timestamp, Some(expected_timestamp));
7315 assert_eq!(state.replication_timestamp, state.replica_timestamp);
7316 assert_eq!(state.replicate_decision_str, "arn=replicate;");
7317 assert_eq!(
7318 state
7319 .reset_statuses_map
7320 .get("arn:minio:replication::TenantA:bucket")
7321 .map(String::as_str),
7322 Some("reset-id")
7323 );
7324 assert_eq!(
7325 state
7326 .target_delete_marker_version_ids
7327 .get("arn:minio:replication::TenantA:bucket")
7328 .map(String::as_str),
7329 Some("target-version")
7330 );
7331 }
7332
7333 #[test]
7334 fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
7335 let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
7336 let mut malformed = HashMap::new();
7337 rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
7338 let malformed_source = ObjectInfo {
7339 delete_marker: true,
7340 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7341 user_defined: Arc::new(malformed),
7342 ..Default::default()
7343 };
7344 assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
7345
7346 let mut conflicted = HashMap::new();
7347 let suffix = format!(
7348 "{}arn:minio:replication::target:bucket",
7349 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7350 );
7351 rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
7352 conflicted.insert(
7353 format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7354 "target-version-b".to_string(),
7355 );
7356 let conflicted_source = ObjectInfo {
7357 user_defined: Arc::new(conflicted),
7358 ..malformed_source.clone()
7359 };
7360 assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
7361
7362 let mut over_cap = HashMap::new();
7363 for index in 0..=1_000 {
7364 let suffix = format!(
7365 "{}arn:minio:replication::target:bucket-{index}",
7366 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7367 );
7368 rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
7369 }
7370 let over_cap_source = ObjectInfo {
7371 user_defined: Arc::new(over_cap),
7372 ..malformed_source
7373 };
7374 assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
7375 }
7376
7377 #[test]
7378 fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
7379 let mut source_metadata = HashMap::new();
7380 rustfs_utils::http::insert_str(
7381 &mut source_metadata,
7382 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7383 ReplicationStatusType::Replica.to_string(),
7384 );
7385 rustfs_utils::http::insert_str(
7386 &mut source_metadata,
7387 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7388 "arn=COMPLETED;".to_string(),
7389 );
7390 let source = ObjectInfo {
7391 delete_marker: true,
7392 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7393 replication_status_internal: Some("arn=COMPLETED;".to_string()),
7394 user_defined: Arc::new(source_metadata),
7395 ..Default::default()
7396 };
7397
7398 let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
7399 .expect("legacy status-only metadata should remain migratable");
7400 let state = target_opts
7401 .delete_replication
7402 .expect("replication state should be reconstructed");
7403 assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7404 assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7405
7406 let mut target_metadata = (*source.user_defined).clone();
7407 let epoch = OffsetDateTime::UNIX_EPOCH
7408 .format(&time::format_description::well_known::Rfc3339)
7409 .unwrap();
7410 rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
7411 rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
7412 assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
7413 }
7414
7415 #[test]
7416 fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
7417 let version_id = Uuid::nil();
7418 let mod_time = OffsetDateTime::UNIX_EPOCH;
7419 let source = ObjectInfo {
7420 version_id: Some(version_id),
7421 delete_marker: true,
7422 mod_time: Some(mod_time),
7423 user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
7424 replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7425 version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7426 ..Default::default()
7427 };
7428
7429 let mut target = source.clone();
7430 target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
7431 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7432
7433 let mut target = source.clone();
7434 target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7435 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7436
7437 let mut target = source.clone();
7438 target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7439 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7440 }
7441
7442 #[test]
7443 fn equivalent_data_movement_delete_marker_rejects_live_object() {
7444 let source = ObjectInfo {
7445 delete_marker: true,
7446 ..Default::default()
7447 };
7448 let target = ObjectInfo {
7449 delete_marker: false,
7450 ..source.clone()
7451 };
7452
7453 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7454 }
7455
7456 #[test]
7457 fn data_movement_delete_marker_resume_accepts_equivalent_target() {
7458 let source = ObjectInfo {
7459 version_id: Some(Uuid::nil()),
7460 delete_marker: true,
7461 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7462 ..Default::default()
7463 };
7464
7465 let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
7466 .expect("equivalent delete marker target should be evaluated");
7467
7468 assert!(should_resume);
7469 }
7470
7471 #[test]
7472 fn data_movement_delete_marker_resume_rejects_source_pool_target() {
7473 let source = ObjectInfo {
7474 version_id: Some(Uuid::nil()),
7475 delete_marker: true,
7476 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7477 ..Default::default()
7478 };
7479
7480 let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
7481 .expect("source-pool target should be rejected before target lookup");
7482
7483 assert!(!should_resume);
7484 }
7485
7486 #[test]
7487 fn data_movement_resume_target_prefers_selected_non_source_pool() {
7488 let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
7489 assert_eq!(target_pool_idx, 2);
7490 }
7491
7492 #[test]
7493 fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
7494 let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
7495 assert_eq!(target_pool_idx, 3);
7496 assert!(should_check_data_movement_resume_target(1, target_pool_idx));
7497 }
7498
7499 #[test]
7500 fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
7501 let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
7502 assert_eq!(target_pool_idx, 1);
7503 }
7504
7505 #[test]
7506 fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
7507 let source = ObjectInfo {
7508 delete_marker: true,
7509 ..Default::default()
7510 };
7511 let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
7512
7513 assert!(matches!(result, Err(Error::SlowDown)));
7514 }
7515
7516 fn tiered_equivalence_source() -> FileInfo {
7517 let version_id = Uuid::nil();
7518 let transition_version_id = Uuid::new_v4();
7519 let mod_time = OffsetDateTime::UNIX_EPOCH;
7520
7521 FileInfo {
7522 version_id: Some(version_id),
7523 size: 1024,
7524 mod_time: Some(mod_time),
7525 checksum: Some(Bytes::from_static(b"checksum")),
7526 transition_status: TRANSITION_COMPLETE.to_string(),
7527 transitioned_objname: "remote/object".to_string(),
7528 transition_tier: "WARM".to_string(),
7529 transition_version_id: Some(transition_version_id),
7530 replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
7531 replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7532 targets: replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
7533 version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7534 purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
7535 ..Default::default()
7536 })),
7537 metadata: HashMap::from([
7538 ("etag".to_string(), "etag-value".to_string()),
7539 ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
7540 (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
7541 ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
7542 ]),
7543 ..Default::default()
7544 }
7545 }
7546
7547 fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
7548 ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
7549 }
7550
7551 #[test]
7552 fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
7553 let source = tiered_equivalence_source();
7554 let target = tiered_equivalence_target(&source);
7555
7556 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7557 }
7558
7559 #[test]
7560 fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
7561 let source = tiered_equivalence_source();
7562 assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
7563
7564 let mut changed_remote = source.clone();
7565 changed_remote.transitioned_objname = "remote/replaced".to_string();
7566 assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
7567
7568 let mut changed_backend = source.clone();
7569 rustfs_utils::http::metadata_compat::insert_str(
7570 &mut changed_backend.metadata,
7571 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7572 rustfs_utils::crypto::hex([9; 32]),
7573 );
7574 assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
7575 }
7576
7577 #[test]
7578 fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
7579 let mut compressed = tiered_equivalence_source();
7580 compressed.size = 600;
7581 rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
7582 rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
7583 let compressed_target = tiered_equivalence_target(&compressed);
7584 assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
7585
7586 let mut encrypted = tiered_equivalence_source();
7587 encrypted.size = 640;
7588 encrypted.metadata.insert(
7589 rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
7590 "key-id".to_string(),
7591 );
7592 encrypted.metadata.insert(
7593 rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
7594 "1024".to_string(),
7595 );
7596 let encrypted_target = tiered_equivalence_target(&encrypted);
7597 assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
7598 }
7599
7600 #[test]
7601 fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
7602 let mut source = tiered_equivalence_source();
7603 let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
7604 source.metadata.insert(
7605 format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7606 source.transition_tier.clone(),
7607 );
7608 let mut target = tiered_equivalence_target(&source);
7609 Arc::make_mut(&mut target.user_defined)
7610 .insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
7611
7612 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7613 }
7614
7615 #[test]
7616 fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
7617 let mut source = tiered_equivalence_source();
7618 source.parts = vec![rustfs_filemeta::ObjectPartInfo {
7619 number: 1,
7620 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
7621 checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
7622 ..Default::default()
7623 }];
7624 rustfs_utils::http::insert_str(
7625 &mut source.metadata,
7626 rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
7627 r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
7628 );
7629 let mut target = tiered_equivalence_target(&source);
7630 Arc::make_mut(&mut target.parts)[0].mod_time = None;
7631 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7632
7633 let mut missing = target;
7634 Arc::make_mut(&mut missing.parts)[0].checksums = None;
7635 assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
7636 }
7637
7638 #[test]
7639 fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
7640 let source = tiered_equivalence_source();
7641 let mut target = tiered_equivalence_target(&source);
7642 target.transitioned_object.name = "remote/target".to_string();
7643
7644 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7645 }
7646
7647 #[test]
7648 fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
7649 let mut source = tiered_equivalence_source();
7650 source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
7651 let mut target = tiered_equivalence_target(&source);
7652 target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7653
7654 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7655 }
7656
7657 #[test]
7658 fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
7659 let source = tiered_equivalence_source();
7660 let mut target = tiered_equivalence_target(&source);
7661 target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));
7662
7663 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7664 }
7665
7666 #[test]
7667 fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
7668 let source = tiered_equivalence_source();
7669 let mut target = tiered_equivalence_target(&source);
7670 target.user_tags = Arc::new("tag=target".to_string());
7671
7672 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7673 }
7674
7675 #[test]
7676 fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
7677 let source = tiered_equivalence_source();
7678 let mut target = tiered_equivalence_target(&source);
7679 target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7680 target.replication_status = ReplicationStatusType::Failed;
7681
7682 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7683 }
7684
7685 #[test]
7686 fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
7687 let source = tiered_equivalence_source();
7688 let mut target = tiered_equivalence_target(&source);
7689 target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7690 target.version_purge_status = VersionPurgeStatusType::Complete;
7691
7692 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7693 }
7694
7695 #[test]
7696 fn data_movement_tiered_resume_accepts_equivalent_target() {
7697 let source = tiered_equivalence_source();
7698 let target = tiered_equivalence_target(&source);
7699
7700 let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
7701 .expect("equivalent tiered target should be evaluated");
7702
7703 assert!(should_resume);
7704 }
7705
7706 #[test]
7707 fn data_movement_tiered_resume_rejects_source_pool_target() {
7708 let source = tiered_equivalence_source();
7709 let target = tiered_equivalence_target(&source);
7710
7711 let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
7712 .expect("source-pool target should be rejected before target lookup");
7713
7714 assert!(!should_resume);
7715 }
7716
7717 #[test]
7718 fn data_movement_tiered_resume_rejects_missing_target() {
7719 let source = FileInfo {
7720 version_id: Some(Uuid::nil()),
7721 size: 1024,
7722 ..Default::default()
7723 };
7724
7725 let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
7726 .expect("missing tiered target should be evaluated");
7727
7728 assert!(!should_resume);
7729 }
7730
7731 #[test]
7732 fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
7733 let info = ObjectInfo::default();
7734 let opts = ObjectOptions::default();
7735
7736 assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none());
7737 }
7738
7739 #[test]
7740 fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() {
7741 let info = ObjectInfo {
7742 delete_marker: true,
7743 ..Default::default()
7744 };
7745 let opts = ObjectOptions::default();
7746
7747 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7748 .expect("delete marker should stop latest-object reads");
7749
7750 assert!(crate::error::is_err_object_not_found(&err));
7751 }
7752
7753 #[test]
7754 fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() {
7755 let info = ObjectInfo {
7756 delete_marker: true,
7757 ..Default::default()
7758 };
7759 let opts = ObjectOptions {
7760 version_id: Some("vid-1".to_string()),
7761 ..Default::default()
7762 };
7763
7764 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7765 .expect("delete marker version reads should be rejected");
7766
7767 assert!(matches!(err, Error::MethodNotAllowed));
7768 }
7769
7770 #[test]
7771 fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() {
7772 let info = ObjectInfo {
7773 delete_marker: true,
7774 ..Default::default()
7775 };
7776 let opts = ObjectOptions {
7777 version_id: Some("vid-1".to_string()),
7778 delete_marker: true,
7779 ..Default::default()
7780 };
7781
7782 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7783 .expect("delete marker lookup should keep not-found semantics");
7784
7785 assert!(crate::error::is_err_object_not_found(&err));
7786 }
7787
7788 #[test]
7789 fn resolve_latest_object_access_returns_live_object_and_pool_idx() {
7790 let info = ObjectInfo::default();
7791 let opts = ObjectOptions::default();
7792
7793 let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap();
7794
7795 assert_eq!(idx, 7);
7796 assert!(!resolved.delete_marker);
7797 }
7798
7799 #[test]
7800 fn resolve_latest_object_access_rejects_delete_marker_without_version_id() {
7801 let info = ObjectInfo {
7802 delete_marker: true,
7803 ..Default::default()
7804 };
7805 let opts = ObjectOptions::default();
7806
7807 let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7808
7809 assert!(crate::error::is_err_object_not_found(&err));
7810 }
7811
7812 #[test]
7813 fn resolve_latest_object_access_rejects_delete_marker_version_read() {
7814 let info = ObjectInfo {
7815 delete_marker: true,
7816 ..Default::default()
7817 };
7818 let opts = ObjectOptions {
7819 version_id: Some("vid-1".to_string()),
7820 ..Default::default()
7821 };
7822
7823 let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7824
7825 assert!(matches!(err, Error::MethodNotAllowed));
7826 }
7827
7828 #[tokio::test]
7829 async fn latest_versioned_delete_creates_a_new_marker_after_a_current_marker() {
7830 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7831 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7832 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7833 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7834 let bucket = "versioned-delete-marker-identity";
7835 let object = "object.bin";
7836 let opts = ObjectOptions {
7837 versioned: true,
7838 ..Default::default()
7839 };
7840
7841 store
7842 .make_bucket(bucket, &MakeBucketOptions::default())
7843 .await
7844 .expect("bucket should be created");
7845
7846 let first = store
7847 .handle_delete_object(bucket, object, opts.clone())
7848 .await
7849 .expect("first versioned delete should create a marker");
7850 let second = store
7851 .handle_delete_object(bucket, object, opts)
7852 .await
7853 .expect("second versioned delete should create a marker");
7854
7855 assert!(first.delete_marker);
7856 assert!(second.delete_marker);
7857 assert_ne!(
7858 first.version_id, second.version_id,
7859 "each latest versioned delete must persist a distinct delete marker"
7860 );
7861
7862 let versions = store.pools[0]
7863 .get_disks_by_key(object)
7864 .load_file_info_versions_exact(bucket, object)
7865 .await
7866 .expect("delete-marker metadata should decode")
7867 .expect("delete-marker metadata should be persisted");
7868 let marker_ids = versions
7869 .versions
7870 .iter()
7871 .filter(|version| version.deleted)
7872 .filter_map(|version| version.version_id)
7873 .collect::<std::collections::HashSet<_>>();
7874 assert!(marker_ids.contains(&first.version_id.expect("first marker should have a version id")));
7875 assert!(marker_ids.contains(&second.version_id.expect("second marker should have a version id")));
7876 assert_eq!(marker_ids.len(), 2, "both delete markers must remain durable versions");
7877 }
7878
7879 #[tokio::test]
7880 async fn versioned_delete_quorum_failure_rolls_back_and_retries_require_quorum() {
7881 for marker_copies in [0, 2, 4] {
7882 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7883 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7884 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7885 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7886 let bucket = "versioned-delete-quorum-retry";
7887 let object = "object.bin";
7888 let opts = ObjectOptions {
7889 versioned: true,
7890 ..Default::default()
7891 };
7892 let put_opts = ObjectOptions {
7893 no_lock: true,
7894 versioned: true,
7895 ..Default::default()
7896 };
7897 store
7898 .make_bucket(bucket, &MakeBucketOptions::default())
7899 .await
7900 .expect("bucket should be created");
7901 let set = store.pools[0].get_disks_by_key(object);
7902 let payload = vec![0x5a; 4096];
7903 let mut reader = PutObjReader::from_vec(payload.clone());
7904 let original = set
7905 .put_object(bucket, object, &mut reader, &put_opts)
7906 .await
7907 .expect("the versioned 4 KiB object should be written");
7908 let disks = set.disks.read().await.clone();
7909 let marker = rustfs_filemeta::FileInfo {
7911 name: object.to_string(),
7912 version_id: Some(Uuid::new_v4()),
7913 deleted: true,
7914 mark_deleted: true,
7915 mod_time: Some(time::OffsetDateTime::now_utc()),
7916 ..Default::default()
7917 };
7918 for disk in disks.iter().take(marker_copies).flatten() {
7919 disk.delete_version(bucket, object, marker.clone(), true, crate::disk::DeleteOptions::default())
7920 .await
7921 .expect("the initial marker should be persisted on the selected disks");
7922 }
7923 let mut baseline = Vec::new();
7924 for disk in disks.iter().flatten() {
7925 baseline.push(
7926 tokio::fs::read(disk.path().join(bucket).join(object).join("xl.meta"))
7927 .await
7928 .expect("initial metadata should exist"),
7929 );
7930 }
7931 {
7932 let mut online = set.disks.write().await;
7933 online[2] = None;
7934 online[3] = None;
7935 }
7936 for attempt in 1..=3 {
7937 let err = store
7938 .handle_delete_object(bucket, object, opts.clone())
7939 .await
7940 .expect_err("every explicit retry must fail with only two of four disks available");
7941 assert!(
7942 matches!(
7943 &err,
7944 StorageError::InsufficientWriteQuorum(error_bucket, error_object)
7945 if error_bucket == bucket && error_object == object
7946 ),
7947 "expected write quorum failure with {marker_copies} marker copies, attempt {attempt}: {err:?}"
7948 );
7949 for (index, disk) in disks.iter().enumerate() {
7950 let metadata_path = disk
7951 .as_ref()
7952 .expect("saved disk should exist")
7953 .path()
7954 .join(bucket)
7955 .join(object);
7956 let actual = tokio::fs::read(metadata_path.join("xl.meta"))
7957 .await
7958 .expect("rollback must preserve readable metadata");
7959 assert_eq!(
7960 actual, baseline[index],
7961 "failed DELETE must restore exact metadata on disk {index}, attempt {attempt}"
7962 );
7963 let mut entries = tokio::fs::read_dir(&metadata_path)
7964 .await
7965 .expect("the object directory should remain readable");
7966 while let Some(entry) = entries.next_entry().await.expect("directory entries should be readable") {
7967 assert!(
7968 entry.file_name() != "xl.meta.bkp" && !entry.path().join("xl.meta.bkp").exists(),
7969 "rollback must consume staged metadata backups on disk {index}"
7970 );
7971 }
7972 }
7973 }
7974 *set.disks.write().await = disks;
7975 if marker_copies != 0 {
7976 continue;
7977 }
7978 let committed = store
7979 .handle_delete_object(bucket, object, opts)
7980 .await
7981 .expect("DELETE should succeed after write quorum recovers");
7982 assert!(committed.delete_marker);
7983 assert_ne!(committed.version_id, marker.version_id);
7984 let mut restored = Vec::new();
7985 set.get_object_reader(
7986 bucket,
7987 object,
7988 None,
7989 HeaderMap::new(),
7990 &ObjectOptions {
7991 version_id: original.version_id.map(|id| id.to_string()),
7992 ..Default::default()
7993 },
7994 )
7995 .await
7996 .expect("the original version must remain readable")
7997 .stream
7998 .read_to_end(&mut restored)
7999 .await
8000 .expect("the original version should stream");
8001 assert_eq!(restored, payload);
8002 }
8003 }
8004
8005 #[test]
8006 fn latest_versioned_delete_marker_creation_excludes_specialized_deletes() {
8007 assert!(latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8008 versioned: true,
8009 ..Default::default()
8010 }));
8011 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8012 versioned: true,
8013 version_id: Some(Uuid::new_v4().to_string()),
8014 ..Default::default()
8015 }));
8016 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8017 versioned: true,
8018 delete_marker: true,
8019 ..Default::default()
8020 }));
8021 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8022 versioned: true,
8023 data_movement: true,
8024 ..Default::default()
8025 }));
8026 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8027 versioned: true,
8028 replication_request: true,
8029 ..Default::default()
8030 }));
8031 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8032 versioned: true,
8033 version_suspended: true,
8034 ..Default::default()
8035 }));
8036 }
8037
8038 #[test]
8039 fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
8040 let opts = ObjectOptions {
8041 versioned: true,
8042 ..Default::default()
8043 };
8044
8045 assert!(should_create_delete_marker_for_missing_object(&opts));
8046 }
8047
8048 #[test]
8049 fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
8050 let version_delete = ObjectOptions {
8051 versioned: true,
8052 version_id: Some("vid-1".to_string()),
8053 ..Default::default()
8054 };
8055 let delete_marker_replication = ObjectOptions {
8056 versioned: true,
8057 delete_marker: true,
8058 ..Default::default()
8059 };
8060 let data_movement = ObjectOptions {
8061 versioned: true,
8062 data_movement: true,
8063 ..Default::default()
8064 };
8065
8066 assert!(!should_create_delete_marker_for_missing_object(&version_delete));
8067 assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
8068 assert!(!should_create_delete_marker_for_missing_object(&data_movement));
8069 }
8070
8071 #[test]
8076 fn should_purge_orphan_dir_on_version_not_found_for_dir_key() {
8077 assert!(
8078 should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "ghost/"),
8079 "the real HTTP delete path yields version-not-found on dir keys and must reach the purge"
8080 );
8081 assert!(
8082 should_purge_orphan_dir_on_missing(
8083 &StorageError::VersionNotFound("bucket".into(), "ghost/".into(), Uuid::nil().to_string()),
8084 "ghost/"
8085 ),
8086 "typed VersionNotFound on a dir key must also reach the purge"
8087 );
8088 }
8089
8090 #[test]
8091 fn should_purge_orphan_dir_on_object_not_found_for_dir_key() {
8092 assert!(should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "ghost/"));
8093 assert!(should_purge_orphan_dir_on_missing(
8094 &StorageError::ObjectNotFound("bucket".into(), "ghost/".into()),
8095 "ghost/"
8096 ));
8097 }
8098
8099 #[test]
8100 fn should_not_purge_orphan_dir_for_regular_key_or_other_errors() {
8101 assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "regular.txt"));
8103 assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "regular.txt"));
8104 assert!(!should_purge_orphan_dir_on_missing(&StorageError::ErasureReadQuorum, "ghost/"));
8106 }
8107
8108 #[test]
8109 fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
8110 let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();
8111
8112 assert_eq!(idx, 3);
8113 }
8114
8115 #[test]
8116 fn resolve_decommission_target_pool_idx_result_wraps_error_context() {
8117 let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object")
8118 .expect_err("expected contextual error");
8119 let rendered = err.to_string();
8120
8121 assert!(rendered.contains("failed to select decommission target pool"), "{rendered}");
8122 assert!(rendered.contains("bucket"), "{rendered}");
8123 assert!(rendered.contains("object"), "{rendered}");
8124 assert!(rendered.contains("boom"), "{rendered}");
8125 }
8126
8127 #[test]
8128 fn resolve_decommission_tiered_object_result_passthrough_ok() {
8129 ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object")
8130 .expect("successful decommission result should pass through");
8131 }
8132
8133 #[test]
8134 fn resolve_decommission_tiered_object_result_wraps_error_context() {
8135 let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object")
8136 .expect_err("expected contextual error");
8137 let rendered = err.to_string();
8138
8139 assert!(rendered.contains("failed to decommission tiered object"), "{rendered}");
8140 assert!(rendered.contains("bucket"), "{rendered}");
8141 assert!(rendered.contains("object"), "{rendered}");
8142 assert!(rendered.contains("boom"), "{rendered}");
8143 }
8144
8145 #[test]
8146 fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() {
8147 let err = ECStore::resolve_decommission_tiered_object_result(
8148 Err(Error::DecommissionCapacityBlocked {
8149 message: "target gate busy".to_string(),
8150 }),
8151 "bucket",
8152 "object",
8153 )
8154 .expect_err("expected contextual error");
8155
8156 assert!(matches!(
8157 crate::data_movement::data_movement_stage_source(&err),
8158 Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy"
8159 ));
8160 }
8161
8162 #[test]
8163 fn version_aware_lookup_opts_enables_version_aware_lookup() {
8164 let opts = ObjectOptions {
8165 version_id: Some("vid-1".to_string()),
8166 ..Default::default()
8167 };
8168
8169 let lookup_opts = version_aware_lookup_opts(&opts, true);
8170
8171 assert!(lookup_opts.no_lock);
8172 assert!(lookup_opts.metadata_chg);
8173 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8174 }
8175
8176 #[test]
8177 fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() {
8178 let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true);
8179
8180 assert!(lookup_opts.no_lock);
8181 assert!(!lookup_opts.metadata_chg);
8182 assert!(lookup_opts.version_id.is_none());
8183 }
8184
8185 #[test]
8186 fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() {
8187 let opts = ObjectOptions {
8188 version_id: Some("vid-1".to_string()),
8189 ..Default::default()
8190 };
8191
8192 let lookup_opts = data_movement_pool_lookup_opts(&opts, false);
8193
8194 assert!(!lookup_opts.no_lock);
8195 assert!(lookup_opts.metadata_chg);
8196 assert!(lookup_opts.skip_decommissioned);
8197 assert!(lookup_opts.skip_rebalancing);
8198 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8199 }
8200
8201 #[test]
8202 fn writer_pool_lookup_opts_skips_rebalance_sources() {
8203 let lookup_opts = writer_pool_lookup_opts(
8204 &ObjectOptions {
8205 version_id: Some("vid-1".to_string()),
8206 ..Default::default()
8207 },
8208 true,
8209 );
8210
8211 assert!(lookup_opts.no_lock);
8212 assert!(lookup_opts.metadata_chg);
8213 assert!(lookup_opts.skip_decommissioned);
8214 assert!(lookup_opts.skip_rebalancing);
8215 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8216 }
8217
8218 #[test]
8219 fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() {
8220 let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true);
8221
8222 assert!(lookup_opts.no_lock);
8223 assert!(!lookup_opts.skip_decommissioned);
8224 assert!(lookup_opts.skip_rebalancing);
8225
8226 let explicit_version = delete_pool_lookup_opts(
8227 &ObjectOptions {
8228 versioned: true,
8229 version_id: Some(uuid::Uuid::new_v4().to_string()),
8230 ..Default::default()
8231 },
8232 true,
8233 );
8234 assert!(!explicit_version.skip_decommissioned);
8235 }
8236
8237 #[test]
8238 fn delete_fans_out_for_unversioned_and_explicit_version_mutations() {
8239 assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1));
8240 assert!(should_delete_from_all_pools(
8241 &ObjectOptions {
8242 versioned: true,
8243 version_id: Some(uuid::Uuid::new_v4().to_string()),
8244 ..Default::default()
8245 },
8246 2,
8247 ));
8248 assert!(!should_delete_from_all_pools(
8249 &ObjectOptions {
8250 versioned: true,
8251 ..Default::default()
8252 },
8253 1,
8254 ));
8255 assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0));
8256 }
8257
8258 #[test]
8259 fn batch_delete_identifies_only_latest_versioned_markers() {
8260 let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test(
8261 s3s::dto::VersioningConfiguration {
8262 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8263 ..Default::default()
8264 },
8265 None,
8266 );
8267 let latest = ObjectToDelete {
8268 object_name: "latest".to_string(),
8269 ..Default::default()
8270 };
8271 assert!(batch_delete_creates_latest_marker(&latest, &versioned));
8272 assert!(!batch_delete_targets_pool(true, Some(1), 0));
8273 assert!(batch_delete_targets_pool(true, Some(1), 1));
8274 assert!(!batch_delete_targets_pool(true, Some(1), 2));
8275
8276 let explicit = ObjectToDelete {
8277 object_name: "explicit".to_string(),
8278 version_id: Some(uuid::Uuid::new_v4()),
8279 ..Default::default()
8280 };
8281 assert!(!batch_delete_creates_latest_marker(&explicit, &versioned));
8282 assert!(batch_delete_targets_pool(false, Some(1), 0));
8283
8284 let unversioned = DeleteReplicationConfigSnapshot::default();
8285 assert!(!batch_delete_creates_latest_marker(&latest, &unversioned));
8286 assert!(batch_delete_targets_pool(false, None, 0));
8287 }
8288
8289 #[test]
8290 fn batch_delete_pool_failures_override_success_in_any_pool_order() {
8291 let success = DeletedObject {
8292 object_name: "object".to_string(),
8293 found: true,
8294 ..Default::default()
8295 };
8296 let source_errors = [
8297 StorageError::ErasureWriteQuorum,
8298 StorageError::NamespaceLockQuorumUnavailable {
8299 mode: "delete_objects_commit",
8300 bucket: "bucket".to_string(),
8301 object: "object".to_string(),
8302 required: 1,
8303 achieved: 0,
8304 },
8305 ];
8306
8307 for source_error in source_errors {
8308 for source_first in [true, false] {
8309 let failed = (DeletedObject::default(), Some(source_error.clone()));
8310 let succeeded = (success.clone(), None);
8311 let pool_results = if source_first {
8312 vec![failed, succeeded]
8313 } else {
8314 vec![succeeded, failed]
8315 };
8316
8317 let (_, error, attempted) =
8318 resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8319
8320 assert!(attempted);
8321 assert_eq!(error, Some(source_error.clone()));
8322 }
8323 }
8324 }
8325
8326 #[test]
8327 fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() {
8328 let success = DeletedObject {
8329 object_name: "object".to_string(),
8330 found: true,
8331 ..Default::default()
8332 };
8333 let missing_errors = [
8334 StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
8335 StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
8336 ];
8337
8338 for missing_error in missing_errors {
8339 let missing = (DeletedObject::default(), Some(missing_error.clone()));
8340 for missing_first in [true, false] {
8341 let succeeded = (success.clone(), None);
8342 let pool_results = if missing_first {
8343 vec![missing.clone(), succeeded]
8344 } else {
8345 vec![succeeded, missing.clone()]
8346 };
8347 let (deleted, error, attempted) =
8348 resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8349
8350 assert!(attempted);
8351 let deleted = deleted.expect("successful pool result should be retained");
8352 assert!(deleted.found);
8353 assert_eq!(deleted.object_name, success.object_name.as_str());
8354 assert!(error.is_none());
8355 }
8356
8357 let missing_only = [missing];
8358 let (_, error, attempted) =
8359 resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error)));
8360 assert!(attempted);
8361 assert_eq!(error, Some(missing_error));
8362 }
8363
8364 let silent_missing = [(DeletedObject::default(), None)];
8365 let (_, error, attempted) =
8366 resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error)));
8367 assert!(attempted);
8368 assert!(error.is_none());
8369 }
8370
8371 #[tokio::test]
8372 async fn lifecycle_null_version_cas_distinguishes_absent_and_replacement_pools() {
8373 use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
8374
8375 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8376 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8377 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8378 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
8379 let bucket = RUSTFS_META_BUCKET;
8380 let suspended_snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
8381 VersioningConfiguration {
8382 status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
8383 ..Default::default()
8384 },
8385 None,
8386 ));
8387 let write_opts = ObjectOptions {
8388 no_lock: true,
8389 version_suspended: true,
8390 ..Default::default()
8391 };
8392 let delete_opts = ObjectOptions {
8393 version_suspended: true,
8394 delete_replication_config_snapshot: Some(Arc::clone(&suspended_snapshot)),
8395 ..Default::default()
8396 };
8397
8398 let success_object = "lifecycle-null-only-in-first-pool";
8399 let mut success_reader = PutObjReader::from_vec(b"single-pool null generation".to_vec());
8400 let success_source = first_set
8401 .put_object(bucket, success_object, &mut success_reader, &write_opts)
8402 .await
8403 .expect("the historical null generation should be written to one pool");
8404 let success_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&success_source)
8405 .expect("the null generation should produce an exact lifecycle target");
8406
8407 let (deleted, errors) = store
8408 .handle_delete_objects(bucket, vec![success_target.clone()], delete_opts.clone())
8409 .await;
8410 assert!(
8411 errors.iter().all(Option::is_none),
8412 "an absent peer pool must not override success: {errors:?}"
8413 );
8414 assert!(deleted[0].found, "the pool containing the exact null generation must report success");
8415
8416 let (deleted, errors) = store
8417 .handle_delete_objects(bucket, vec![success_target], delete_opts.clone())
8418 .await;
8419 assert!(
8420 errors.iter().all(Option::is_none),
8421 "an exact lifecycle delete retry must be idempotent: {errors:?}"
8422 );
8423 assert!(!deleted[0].found, "the retried generation should already be absent from every pool");
8424
8425 let mismatch_object = "lifecycle-null-replaced-in-second-pool";
8426 let mut original_reader = PutObjReader::from_vec(b"original null generation".to_vec());
8427 let original = first_set
8428 .put_object(bucket, mismatch_object, &mut original_reader, &write_opts)
8429 .await
8430 .expect("the original null generation should be written to the first pool");
8431 let stale_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&original)
8432 .expect("the original null generation should produce an exact lifecycle target");
8433 let mut replacement_reader = PutObjReader::from_vec(b"replacement null generation".to_vec());
8434 let replacement = second_set
8435 .put_object(bucket, mismatch_object, &mut replacement_reader, &write_opts)
8436 .await
8437 .expect("the replacement null generation should be written to the second pool");
8438 assert_ne!(original.data_dir, replacement.data_dir);
8439
8440 let (_deleted, errors) = store.handle_delete_objects(bucket, vec![stale_target], delete_opts).await;
8441 assert!(
8442 matches!(errors.as_slice(), [Some(StorageError::PreconditionFailed)]),
8443 "a different null generation in any pool must fail the aggregate CAS: {errors:?}"
8444 );
8445 let retained = second_set
8446 .get_object_info(
8447 bucket,
8448 mismatch_object,
8449 &ObjectOptions {
8450 version_id: Some(Uuid::nil().to_string()),
8451 version_suspended: true,
8452 no_lock: true,
8453 ..Default::default()
8454 },
8455 )
8456 .await
8457 .expect("the replacement null generation must remain after a stale CAS");
8458 assert_eq!(retained.data_dir, replacement.data_dir);
8459 }
8460
8461 #[test]
8462 fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
8463 let lookup_opts = data_movement_pool_lookup_opts(
8464 &ObjectOptions {
8465 version_id: Some("vid-1".to_string()),
8466 ..Default::default()
8467 },
8468 true,
8469 );
8470
8471 assert!(lookup_opts.no_lock);
8472 assert!(lookup_opts.metadata_chg);
8473 assert!(lookup_opts.skip_decommissioned);
8474 assert!(lookup_opts.skip_rebalancing);
8475 }
8476
8477 #[test]
8478 fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() {
8479 let mut opts = ObjectOptions {
8480 data_movement: true,
8481 ..Default::default()
8482 };
8483 assert!(!uses_data_movement_pool_selection(&opts));
8484
8485 DecommissionCapacityOwner {
8486 source_pool_index: 1,
8487 operation_id: Uuid::new_v4(),
8488 generation: 2,
8489 owner_nonce: Uuid::new_v4(),
8490 mutation_id: None,
8491 }
8492 .apply_to(&mut opts);
8493
8494 assert!(uses_data_movement_pool_selection(&opts));
8495 }
8496
8497 #[test]
8498 fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
8499 let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8500 no_lock: false,
8501 skip_decommissioned: false,
8502 ..Default::default()
8503 });
8504
8505 assert!(lookup_opts.skip_decommissioned);
8506 assert!(lookup_opts.skip_rebalancing);
8507 assert!(!lookup_opts.no_lock);
8508 }
8509
8510 #[test]
8511 fn transition_restore_pool_opts_rejects_ambient_no_lock() {
8512 let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8513 no_lock: true,
8514 ..Default::default()
8515 });
8516
8517 assert!(lookup_opts.skip_decommissioned);
8518 assert!(lookup_opts.skip_rebalancing);
8519 assert!(!lookup_opts.no_lock);
8520 }
8521
8522 #[test]
8523 fn delete_objects_lock_names_are_sorted_and_unique() {
8524 let objects = vec![
8525 ObjectToDelete {
8526 object_name: "beta".to_string(),
8527 ..Default::default()
8528 },
8529 ObjectToDelete {
8530 object_name: "alpha".to_string(),
8531 ..Default::default()
8532 },
8533 ObjectToDelete {
8534 object_name: "beta".to_string(),
8535 ..Default::default()
8536 },
8537 ];
8538
8539 assert_eq!(sorted_unique_delete_object_names(&objects), vec!["alpha", "beta"]);
8540 }
8541
8542 async fn new_read_lock_test_store() -> ECStore {
8543 let format = FormatV3::new(1, 2);
8544 let endpoints = vec![
8545 Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
8546 Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
8547 ];
8548 let pool_endpoints = PoolEndpoints {
8549 legacy: false,
8550 set_count: 1,
8551 drives_per_set: 2,
8552 endpoints: Endpoints::from(endpoints),
8553 cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
8554 platform: "test".to_string(),
8555 };
8556 let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
8557 let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
8558 .await
8559 .expect("test sets should be created with empty disks");
8560
8561 ECStore {
8562 id: Uuid::new_v4(),
8563 disk_map: HashMap::new(),
8564 pools: vec![sets],
8565 peer_sys: S3PeerSys::new(&endpoint_pools),
8566 pool_meta: RwLock::new(PoolMeta::default()),
8567 rebalance_meta: RwLock::new(None),
8568 decommission_cancelers: RwLock::new(Vec::new()),
8569 start_gate: Mutex::new(()),
8570 pool_meta_save_gate: Mutex::default(),
8571 ctx: crate::runtime::instance::bootstrap_ctx(),
8572 bucket_fence_registry: std::sync::Arc::default(),
8573 }
8574 }
8575
8576 #[tokio::test]
8577 async fn delete_object_handler_futures_remain_stack_bounded() {
8578 let store = new_read_lock_test_store().await;
8579 let unified_future = store.handle_delete_object_with_journal("bucket", "object", ObjectOptions::default(), None);
8580 let unified_future_size = std::mem::size_of_val(&unified_future);
8581
8582 assert!(
8583 unified_future_size <= 4 * 1024,
8584 "unified delete handler future must remain stack-bounded; measured {unified_future_size} bytes"
8585 );
8586 drop(unified_future);
8587
8588 let batch_future =
8589 store.handle_delete_objects_with_journal_and_accounting("bucket", Vec::new(), ObjectOptions::default(), None);
8590 let batch_future_size = std::mem::size_of_val(&batch_future);
8591 assert!(
8592 batch_future_size <= 4 * 1024,
8593 "batch delete handler future must remain stack-bounded; measured {batch_future_size} bytes"
8594 );
8595 drop(batch_future);
8596
8597 let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default());
8598 let outer_future_size = std::mem::size_of_val(&outer_future);
8599 assert!(
8600 outer_future_size <= 16 * 1024,
8601 "outer delete handler future must remain stack-bounded; measured {outer_future_size} bytes"
8602 );
8603 }
8604
8605 async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
8606 new_prepared_reader_test_store_with_ctx(set_disks, crate::runtime::instance::bootstrap_ctx()).await
8607 }
8608
8609 async fn new_prepared_reader_test_store_with_ctx(
8610 set_disks: &[Arc<SetDisks>],
8611 ctx: Arc<crate::runtime::instance::InstanceContext>,
8612 ) -> ECStore {
8613 let mut pool_configs = Vec::with_capacity(set_disks.len());
8614 let mut pools = Vec::with_capacity(set_disks.len());
8615
8616 for (pool_idx, set_disks) in set_disks.iter().enumerate() {
8617 let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
8618 for endpoint in endpoints.as_mut() {
8619 endpoint.set_pool_index(pool_idx);
8620 }
8621 let pool_config = PoolEndpoints {
8622 legacy: false,
8623 set_count: 1,
8624 drives_per_set: set_disks.set_drive_count,
8625 endpoints,
8626 cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
8627 platform: "test".to_string(),
8628 };
8629 let disks = set_disks.disks.read().await.clone();
8630 let pool = Sets::new_with_instance_ctx(
8631 disks,
8632 &pool_config,
8633 &set_disks.format,
8634 pool_idx,
8635 set_disks.default_parity_count,
8636 Arc::clone(&ctx),
8637 )
8638 .await
8639 .expect("prepared-reader test pool should be created from local disks");
8640 pool_configs.push(pool_config);
8641 pools.push(pool);
8642 }
8643
8644 new_prepared_reader_test_store_from_pools(pools, pool_configs, ctx)
8645 }
8646
8647 fn new_prepared_reader_test_store_from_pools(
8648 pools: Vec<Arc<Sets>>,
8649 pool_configs: Vec<PoolEndpoints>,
8650 ctx: Arc<crate::runtime::instance::InstanceContext>,
8651 ) -> ECStore {
8652 let endpoint_pools = EndpointServerPools::from(pool_configs);
8653 ECStore {
8654 id: Uuid::new_v4(),
8655 disk_map: HashMap::new(),
8656 pools,
8657 peer_sys: S3PeerSys::new_with_instance_ctx(&endpoint_pools, Arc::clone(&ctx)),
8658 pool_meta: RwLock::new(PoolMeta::default()),
8659 rebalance_meta: RwLock::new(None),
8660 decommission_cancelers: RwLock::new(Vec::new()),
8661 start_gate: Mutex::new(()),
8662 pool_meta_save_gate: Mutex::default(),
8663 ctx,
8664 bucket_fence_registry: std::sync::Arc::default(),
8665 }
8666 }
8667
8668 #[tokio::test]
8669 #[serial_test::serial]
8670 async fn object_lock_diagnostics_report_historical_get_blocked_by_put_commit() {
8671 use crate::diagnostics::object_lock::tests::CapturedLockEvents;
8672 use crate::object_api::WriteCompletion;
8673 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
8674 use tracing::{Instrument, instrument::WithSubscriber};
8675
8676 let capture = CapturedLockEvents::default();
8677 temp_env::async_with_vars(
8678 [
8679 (rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true")),
8680 (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
8681 ],
8682 async {
8683 let bucket = "historical-get-lock-diagnostics";
8684 let object = "object.bin";
8685 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8686 let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, ctx.clone()).await;
8687 let store = new_prepared_reader_test_store_with_ctx(&[original], ctx).await;
8688 let set = store.pools[0].get_disks_by_key(object);
8689 set.make_bucket(bucket, &MakeBucketOptions::default())
8690 .await
8691 .expect("create bucket");
8692 let opts = ObjectOptions {
8693 versioned: true,
8694 write_completion: WriteCompletion::TailDrained,
8695 ..Default::default()
8696 };
8697 let mut versions = Vec::new();
8698 for value in [0x11, 0x22, 0x33] {
8699 let info = set
8700 .put_object(bucket, object, &mut PutObjReader::from_vec(vec![value; 8193]), &opts)
8701 .await
8702 .expect("seed version");
8703 versions.push((info.version_id.expect("explicit seed version"), value));
8704 }
8705
8706 let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
8707 let put_set = set.clone();
8708 let put = tokio::spawn(
8709 async move {
8710 put_set
8711 .put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x44; 8193]), &opts)
8712 .await
8713 }
8714 .instrument(tracing::info_span!("request", request_id = "diagnostic-put"))
8715 .with_current_subscriber(),
8716 );
8717 barrier.wait_until_paused().await;
8718
8719 let historical_version = versions[0].0.to_string();
8720 let result = store
8721 .prepare_get_object_reader(
8722 bucket,
8723 object,
8724 None,
8725 HeaderMap::new(),
8726 &ObjectOptions {
8727 version_id: Some(historical_version.clone()),
8728 versioned: true,
8729 ..Default::default()
8730 },
8731 )
8732 .instrument(tracing::info_span!("request", request_id = "diagnostic-get"))
8733 .await;
8734 assert!(
8735 matches!(result, Err(StorageError::Lock(rustfs_lock::LockError::Timeout { .. }))),
8736 "a held PUT commit must preserve the typed GET lock timeout"
8737 );
8738 let rows = capture.rows();
8739 let failure = rows
8740 .iter()
8741 .find(|row| row["fields"]["op"] == "prepare_get_object" && row["fields"]["state"] == "failed")
8742 .expect("failed GET acquisition must be observable before error propagation");
8743 assert_eq!(failure["fields"]["requested_version_id"], historical_version);
8744 assert_eq!(failure["fields"]["failure"], "timeout");
8745 assert_eq!(failure["fields"]["timeout_ms"], 1000);
8746 assert!(failure["fields"]["acquire_ms"].as_u64().expect("wait duration") >= 1000);
8747 assert!(
8748 failure["spans"]
8749 .as_array()
8750 .expect("request spans")
8751 .iter()
8752 .any(|span| span["request_id"] == "diagnostic-get")
8753 );
8754 let put_acquired = rows
8755 .iter()
8756 .rev()
8757 .find(|row| row["fields"]["op"] == "put_object_commit" && row["fields"]["state"] == "acquired")
8758 .expect("PUT lock holder event");
8759 assert_ne!(put_acquired["fields"]["lock_attempt_id"], failure["fields"]["lock_attempt_id"]);
8760 assert!(
8761 !rows
8762 .iter()
8763 .any(|row| row["fields"]["lock_attempt_id"] == put_acquired["fields"]["lock_attempt_id"]
8764 && row["fields"]["state"] == "guard_dropped"),
8765 "a paused commit still owns the lock"
8766 );
8767
8768 barrier.release();
8769 let new_version = put
8770 .await
8771 .expect("PUT task")
8772 .expect("new version commits")
8773 .version_id
8774 .expect("new version id");
8775 versions.push((new_version, 0x44));
8776 for (version_id, value) in versions {
8777 let prepared = store
8778 .prepare_get_object_reader(
8779 bucket,
8780 object,
8781 None,
8782 HeaderMap::new(),
8783 &ObjectOptions {
8784 version_id: Some(version_id.to_string()),
8785 versioned: true,
8786 ..Default::default()
8787 },
8788 )
8789 .await
8790 .expect("every acknowledged version remains readable");
8791 assert_eq!(prepared.object_info().version_id, Some(version_id));
8792 let mut reader = prepared.into_reader().await.expect("open historical body");
8793 let mut body = Vec::new();
8794 reader
8795 .stream
8796 .read_to_end(&mut body)
8797 .await
8798 .expect("read complete version body");
8799 assert_eq!(body, vec![value; 8193]);
8800 }
8801 }
8802 .with_subscriber(capture.subscriber()),
8803 )
8804 .await;
8805 }
8806
8807 async fn multipool_version_test_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
8808 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8809 let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8810 let (second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8811 dirs.extend(second_dirs);
8812 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first_set, second_set], ctx).await);
8813 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
8814 store
8815 .handle_make_bucket(
8816 bucket,
8817 &MakeBucketOptions {
8818 versioning_enabled: true,
8819 ..Default::default()
8820 },
8821 )
8822 .await
8823 .expect("create the versioned bucket in both pools");
8824 (dirs, store)
8825 }
8826
8827 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8828 async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
8829 let bucket = "retired-marker-c11";
8830 let object = "marker.bin";
8831 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8832 let (dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
8833 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
8834 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8835 let create = MakeBucketOptions {
8836 versioning_enabled: true,
8837 ..Default::default()
8838 };
8839 store.handle_make_bucket(bucket, &create).await.expect("create old bucket");
8840 let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old identity");
8841 let mut historical = Vec::new();
8842 for value in 1..=3 {
8843 historical.push(
8844 store
8845 .put_object(
8846 bucket,
8847 "history.bin",
8848 &mut PutObjReader::from_vec(vec![value; 4097]),
8849 &ObjectOptions {
8850 versioned: true,
8851 ..Default::default()
8852 },
8853 )
8854 .await
8855 .expect("write old version")
8856 .version_id
8857 .unwrap(),
8858 );
8859 }
8860 let marker = store
8861 .delete_object(
8862 bucket,
8863 object,
8864 ObjectOptions {
8865 versioned: true,
8866 ..Default::default()
8867 },
8868 )
8869 .await
8870 .expect("create old marker")
8871 .version_id
8872 .expect("explicit marker version");
8873 assert!(
8874 store
8875 .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8876 .await
8877 .is_err()
8878 );
8879 assert!(
8880 !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8881 .await
8882 .unwrap(),
8883 "failed bucket deletion cannot publish retirement"
8884 );
8885 let set = &store.pools[0].disk_set[0];
8886 let before = set.disks.read().await.clone();
8887 for disk in before.iter().flatten() {
8888 let info = disk
8889 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
8890 .await
8891 .expect("read original marker");
8892 assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation));
8893 }
8894 for slot in 12..16 {
8897 set.disks.write().await[slot] = None;
8898 }
8899 for version in historical {
8900 store
8901 .delete_object(
8902 bucket,
8903 "history.bin",
8904 ObjectOptions {
8905 versioned: true,
8906 version_id: Some(version.to_string()),
8907 ..Default::default()
8908 },
8909 )
8910 .await
8911 .expect("delete exact old data version with four drives offline");
8912 }
8913 store
8914 .delete_object(
8915 bucket,
8916 object,
8917 ObjectOptions {
8918 versioned: true,
8919 version_id: Some(marker.to_string()),
8920 ..Default::default()
8921 },
8922 )
8923 .await
8924 .expect("delete exact old marker with four drives offline");
8925 store
8926 .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8927 .await
8928 .expect("delete empty old bucket");
8929 assert!(
8930 crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8931 .await
8932 .expect("durable retirement")
8933 );
8934 store
8935 .handle_make_bucket(bucket, &create)
8936 .await
8937 .expect("recreate same bucket name");
8938 let current = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
8939 assert_ne!(old_incarnation, current);
8940 let mut new_versions = Vec::new();
8941 for (key, payload) in [
8942 (object, b"new generation reused key".as_slice()),
8943 ("new.bin", b"new generation control".as_slice()),
8944 ] {
8945 let info = store
8946 .put_object(
8947 bucket,
8948 key,
8949 &mut PutObjReader::from_vec(payload.to_vec()),
8950 &ObjectOptions {
8951 versioned: true,
8952 ..Default::default()
8953 },
8954 )
8955 .await
8956 .expect("write new generation");
8957 new_versions.push((key, info.version_id.unwrap(), payload));
8958 }
8959 *set.disks.write().await = before.clone();
8960 let heal = rustfs_heal_contracts::heal_channel::HealOpts {
8961 remove: true,
8962 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
8963 ..Default::default()
8964 };
8965 let observed = store
8966 .clone()
8967 .inner_list_object_versions(bucket, "", None, None, None, 100)
8968 .await
8969 .expect("list after rejoin");
8970 assert_eq!(observed.objects.len(), 2, "stale marker must not contaminate listing");
8971 assert!(observed.objects.iter().all(|info| !info.delete_marker));
8972 for (key, version, _) in &new_versions {
8973 let repaired = store
8974 .heal_object_with_proof(bucket, key, &version.to_string(), &heal)
8975 .await
8976 .expect("heal new generation");
8977 assert!(repaired.error.is_none(), "new version heal: {:?}", repaired.error);
8978 }
8979 let no_remove = store
8980 .heal_object_with_proof(
8981 bucket,
8982 object,
8983 &marker.to_string(),
8984 &rustfs_heal_contracts::heal_channel::HealOpts { remove: false, ..heal },
8985 )
8986 .await
8987 .expect("inspect marker without removal");
8988 assert!(no_remove.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
8989 assert!(no_remove.absence.is_none());
8990 let dry = store
8991 .heal_object_with_proof(
8992 bucket,
8993 object,
8994 &marker.to_string(),
8995 &rustfs_heal_contracts::heal_channel::HealOpts { dry_run: true, ..heal },
8996 )
8997 .await
8998 .expect("dry run");
8999 assert!(dry.absence.is_none());
9000 for disk in before.iter().skip(12).flatten() {
9001 assert!(
9002 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9003 .await
9004 .is_ok()
9005 );
9006 }
9007 let original_marker = before[12]
9008 .as_ref()
9009 .unwrap()
9010 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9011 .await
9012 .unwrap();
9013 for stamp in [None, Some(current), Some(Uuid::new_v4())] {
9014 for disk in before.iter().skip(12).flatten() {
9015 let mut changed = original_marker.clone();
9016 changed.metadata.clear();
9017 if let Some(stamp) = stamp {
9018 changed.set_delete_marker_incarnation(stamp);
9019 }
9020 disk.write_metadata("", bucket, object, changed).await.unwrap();
9021 }
9022 let unresolved = store
9023 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9024 .await
9025 .unwrap();
9026 assert!(
9027 unresolved.error.as_ref().is_some_and(Error::is_retired_marker_deferred),
9028 "legacy/current/unproven markers must be deferred: {:?}",
9029 unresolved.error
9030 );
9031 assert!(unresolved.absence.is_none());
9032 for disk in before.iter().skip(12).flatten() {
9033 assert!(
9034 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9035 .await
9036 .is_ok()
9037 );
9038 disk.write_metadata("", bucket, object, original_marker.clone())
9039 .await
9040 .unwrap();
9041 }
9042 }
9043 set.disks.write().await[15] = None;
9044 let incomplete = store
9045 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9046 .await
9047 .unwrap();
9048 assert!(incomplete.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9049 assert!(incomplete.absence.is_none(), "offline targets cannot be omitted from a cleanup receipt");
9050 set.disks.write().await[15] = before[15].clone();
9051 let failure =
9052 crate::set_disk::DanglingDeleteFailure::install(bucket, object, 15, crate::disk::error::DiskError::FaultyDisk);
9053 let partial = store
9054 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9055 .await
9056 .unwrap();
9057 assert!(partial.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9058 assert!(
9059 partial.absence.is_none(),
9060 "fifteen successful/absent disks cannot hide one remaining marker"
9061 );
9062 assert!(
9063 before[15]
9064 .as_ref()
9065 .unwrap()
9066 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9067 .await
9068 .is_ok()
9069 );
9070 drop(failure);
9071 let cleaned = store
9072 .heal_object_at_incarnation(bucket, object, &marker.to_string(), current, &heal)
9073 .await
9074 .expect("cleanup retired marker");
9075 assert!(cleaned.error.is_none(), "cleanup error: {:?}", cleaned.error);
9076 let proof = cleaned.absence.expect("complete cleanup receipt");
9077 assert!(proof.removed);
9078 assert_eq!(proof.bucket_incarnation_id, current);
9079 for disk in before.iter().flatten() {
9080 assert!(matches!(
9081 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9082 .await,
9083 Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9084 ));
9085 }
9086 let replay = store
9087 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9088 .await
9089 .expect("idempotent replay");
9090 assert!(replay.error.is_none());
9091 assert!(!replay.absence.expect("authoritative absence receipt").removed);
9092 for (key, version, payload) in new_versions {
9093 let mut reader = store
9094 .handle_get_object_reader(
9095 bucket,
9096 key,
9097 None,
9098 HeaderMap::new(),
9099 &ObjectOptions {
9100 version_id: Some(version.to_string()),
9101 ..Default::default()
9102 },
9103 )
9104 .await
9105 .expect("new generation still readable");
9106 let mut actual = Vec::new();
9107 reader.stream.read_to_end(&mut actual).await.expect("read new bytes");
9108 assert_eq!(actual, payload);
9109 }
9110 assert_eq!(dirs.len(), 16);
9111 }
9112
9113 #[tokio::test]
9114 async fn multipool_delete_marker_stays_with_existing_versions() {
9115 let bucket = "multipool-marker-routing";
9116 let object = "history.bin";
9117 let (_dirs, store) = multipool_version_test_store(bucket).await;
9118
9119 let mut expected_versions = Vec::new();
9120 for value in 1..=3_u8 {
9121 let written = store.pools[1]
9122 .put_object(
9123 bucket,
9124 object,
9125 &mut PutObjReader::from_vec(vec![value; 4097]),
9126 &ObjectOptions {
9127 versioned: true,
9128 user_defined: HashMap::from([
9129 (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), format!("generation={value}")),
9130 ("x-amz-meta-generation".to_string(), value.to_string()),
9131 ]),
9132 ..Default::default()
9133 },
9134 )
9135 .await
9136 .expect("write a historical version deterministically to pool 1");
9137 expected_versions.push(written.version_id.expect("versioned PUT must acknowledge a UUID"));
9138 }
9139 let marker = store
9140 .delete_object(
9141 bucket,
9142 object,
9143 ObjectOptions {
9144 versioned: true,
9145 ..Default::default()
9146 },
9147 )
9148 .await
9149 .expect("delete the current version");
9150 assert!(marker.delete_marker);
9151 let marker_id = marker.version_id.expect("DELETE must acknowledge a marker UUID");
9152 assert!(!expected_versions.contains(&marker_id));
9153
9154 let local = store.pools[1]
9155 .clone()
9156 .inner_list_object_versions(bucket, object, None, None, None, 10)
9157 .await
9158 .expect("list the object-owning pool");
9159 assert_eq!(local.objects.len(), 4, "the marker must be committed beside the three existing versions");
9160 assert_eq!(local.objects[0].version_id, Some(marker_id));
9161 assert!(local.objects[0].delete_marker && local.objects[0].is_latest);
9162 let versions = store
9163 .clone()
9164 .inner_list_object_versions(bucket, object, None, None, None, 10)
9165 .await
9166 .expect("list all pools");
9167 assert_eq!(versions.objects.len(), 4);
9168 assert_eq!(versions.objects.iter().filter(|version| version.is_latest).count(), 1);
9169 for (index, version_id) in expected_versions.iter().copied().enumerate() {
9170 assert!(
9171 versions
9172 .objects
9173 .iter()
9174 .any(|version| version.version_id == Some(version_id) && !version.is_latest)
9175 );
9176 let mut reader = store
9177 .handle_get_object_reader(
9178 bucket,
9179 object,
9180 None,
9181 HeaderMap::new(),
9182 &ObjectOptions {
9183 version_id: Some(version_id.to_string()),
9184 ..Default::default()
9185 },
9186 )
9187 .await
9188 .expect("historical version should remain readable");
9189 let mut payload = Vec::new();
9190 reader
9191 .stream
9192 .read_to_end(&mut payload)
9193 .await
9194 .expect("read all historical bytes");
9195 let value = u8::try_from(index + 1).expect("fixture generation fits u8");
9196 assert_eq!(payload, vec![value; 4097]);
9197 let listed = versions
9198 .objects
9199 .iter()
9200 .find(|version| version.version_id == Some(version_id))
9201 .expect("listed historical version");
9202 assert_eq!(listed.user_tags.as_str(), format!("generation={value}"));
9203 assert_eq!(listed.user_defined.get("x-amz-meta-generation"), Some(&value.to_string()));
9204 }
9205
9206 let repeated = store
9207 .delete_object(
9208 bucket,
9209 object,
9210 ObjectOptions {
9211 versioned: true,
9212 ..Default::default()
9213 },
9214 )
9215 .await
9216 .expect("repeat simple DELETE");
9217 assert!(repeated.delete_marker);
9218 assert_ne!(
9219 repeated.version_id,
9220 Some(marker_id),
9221 "each enabled-versioning DELETE creates a new marker"
9222 );
9223 store
9224 .delete_object(
9225 bucket,
9226 object,
9227 ObjectOptions {
9228 versioned: true,
9229 version_id: repeated.version_id.map(|id| id.to_string()),
9230 ..Default::default()
9231 },
9232 )
9233 .await
9234 .expect("remove the newest marker by identity");
9235 store
9236 .delete_object(
9237 bucket,
9238 object,
9239 ObjectOptions {
9240 versioned: true,
9241 version_id: Some(marker_id.to_string()),
9242 ..Default::default()
9243 },
9244 )
9245 .await
9246 .expect("remove the original marker by identity");
9247 let current = store
9248 .get_object_info(bucket, object, &ObjectOptions::default())
9249 .await
9250 .expect("previous version becomes current");
9251 assert_eq!(current.version_id, expected_versions.last().copied());
9252 assert!(!current.delete_marker);
9253 }
9254
9255 #[tokio::test]
9256 async fn multipool_existing_split_history_lists_and_paginates_without_metadata_writes() {
9257 for marker_pool in 0..2 {
9258 let bucket = format!("multipool-split-history-{marker_pool}");
9259 let object = "history.bin";
9260 let (dirs, store) = multipool_version_test_store(&bucket).await;
9261 let mut expected = Vec::new();
9262 for value in 1..=3_u8 {
9263 let version = store.pools[1 - marker_pool]
9264 .put_object(
9265 &bucket,
9266 object,
9267 &mut PutObjReader::from_vec(vec![value; 4097]),
9268 &ObjectOptions {
9269 versioned: true,
9270 ..Default::default()
9271 },
9272 )
9273 .await
9274 .expect("seed history through the owning pool's normal write path");
9275 expected.push(version.version_id);
9276 }
9277 let marker = store.pools[marker_pool]
9278 .delete_object(
9279 &bucket,
9280 object,
9281 ObjectOptions {
9282 versioned: true,
9283 ..Default::default()
9284 },
9285 )
9286 .await
9287 .expect("reproduce the previously committed split marker with a normal pool DELETE");
9288 assert!(marker.delete_marker);
9289 expected.push(marker.version_id);
9290 expected.reverse();
9291 let mut before = Vec::new();
9292 for dir in &dirs {
9293 before.push(
9294 tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9295 .await
9296 .expect("snapshot persisted version metadata"),
9297 );
9298 }
9299 for max_keys in [1, 2, 4, 10] {
9300 let mut key_marker = None;
9301 let mut version_marker = None;
9302 let mut listed = Vec::new();
9303 let mut completed = false;
9304 for _ in 0..6 {
9305 let page = store
9306 .clone()
9307 .inner_list_object_versions(&bucket, object, key_marker.clone(), version_marker.clone(), None, max_keys)
9308 .await
9309 .expect("read a complete merged version page");
9310 listed.extend(page.objects);
9311 if !page.is_truncated {
9312 completed = true;
9313 break;
9314 }
9315 assert_ne!(
9316 (&page.next_marker, &page.next_version_idmarker),
9317 (&key_marker, &version_marker),
9318 "version cursor must advance"
9319 );
9320 key_marker = page.next_marker;
9321 version_marker = page.next_version_idmarker;
9322 }
9323 assert!(completed, "pagination must terminate");
9324 assert_eq!(listed.iter().map(|version| version.version_id).collect::<Vec<_>>(), expected);
9325 assert!(listed[0].delete_marker && listed[0].is_latest);
9326 assert!(listed[1..].iter().all(|version| !version.delete_marker && !version.is_latest));
9327 }
9328 let visible = store
9329 .clone()
9330 .list_objects_generic(&bucket, "", None, None, 10, false)
9331 .await
9332 .expect("list current objects");
9333 assert!(visible.objects.is_empty(), "the global current marker hides the object");
9334 for (dir, before) in dirs.iter().zip(before) {
9335 assert_eq!(
9336 tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9337 .await
9338 .expect("read unchanged metadata"),
9339 before
9340 );
9341 }
9342 }
9343 }
9344
9345 #[tokio::test]
9346 async fn multipool_conflicting_version_metadata_fails_the_listing_request() {
9347 let bucket = "multipool-version-conflict";
9348 let (_dirs, store) = multipool_version_test_store(bucket).await;
9349 store.pools[0]
9350 .put_object(
9351 bucket,
9352 "a.bin",
9353 &mut PutObjReader::from_vec(b"preceding result".to_vec()),
9354 &ObjectOptions {
9355 versioned: true,
9356 ..Default::default()
9357 },
9358 )
9359 .await
9360 .expect("seed an entry before the conflict");
9361 let version_id = Uuid::new_v4();
9362 let mod_time = OffsetDateTime::now_utc();
9363 let mut etags = Vec::new();
9364 for (pool_idx, value) in [(0, 1), (1, 2)] {
9365 let written = store.pools[pool_idx]
9366 .put_object(
9367 bucket,
9368 "z.bin",
9369 &mut PutObjReader::from_vec(vec![value; 4097]),
9370 &ObjectOptions {
9371 versioned: true,
9372 version_id: Some(version_id.to_string()),
9373 mod_time: Some(mod_time),
9374 ..Default::default()
9375 },
9376 )
9377 .await
9378 .expect("persist independent conflicting copies");
9379 assert_eq!(written.version_id, Some(version_id));
9380 etags.push(written.etag);
9381 }
9382 assert_ne!(etags[0], etags[1], "fixture must contain a semantic conflict");
9383 let err = store
9384 .clone()
9385 .inner_list_object_versions(bucket, "", None, None, None, 10)
9386 .await
9387 .expect_err("partial output must not hide the merge error");
9388 assert_eq!(err, StorageError::FileCorrupt);
9389
9390 let cancellation = tokio_util::sync::CancellationToken::new();
9391 let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
9392 let walk = store
9393 .clone()
9394 .walk(cancellation.clone(), bucket, "", sender, WalkOptions::default());
9395 let drain = async { while receiver.recv().await.is_some() {} };
9396 let (walk_result, ()) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(walk, drain) })
9397 .await
9398 .expect("bounded walk output must drain and terminate on a merge error");
9399 assert_eq!(
9400 walk_result.expect_err("walk must report the same metadata conflict"),
9401 StorageError::FileCorrupt
9402 );
9403 assert!(!cancellation.is_cancelled(), "worker failure must not cancel the caller's request");
9404 }
9405
9406 #[tokio::test]
9407 async fn multipool_marker_rejects_unwritable_owner_without_falling_back() {
9408 let bucket = "multipool-unwritable-owner";
9409 let object = "history.bin";
9410 let (_dirs, store) = multipool_version_test_store(bucket).await;
9411 store.pools[1]
9412 .put_object(
9413 bucket,
9414 object,
9415 &mut PutObjReader::from_vec(vec![1; 4097]),
9416 &ObjectOptions {
9417 versioned: true,
9418 ..Default::default()
9419 },
9420 )
9421 .await
9422 .expect("seed the nonzero owner");
9423 *store.pool_meta.write().await = PoolMeta {
9424 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
9425 ..Default::default()
9426 };
9427 let err = store
9428 .delete_object(
9429 bucket,
9430 object,
9431 ObjectOptions {
9432 versioned: true,
9433 ..Default::default()
9434 },
9435 )
9436 .await
9437 .expect_err("suspended owner must reject marker creation");
9438 assert_eq!(err, StorageError::SlowDown);
9439 *store.pool_meta.write().await = PoolMeta::default();
9440
9441 let mut rebalancing = crate::services::rebalance::RebalanceStats {
9442 participating: true,
9443 ..Default::default()
9444 };
9445 rebalancing.info.status = crate::services::rebalance::RebalStatus::Started;
9446 *store.rebalance_meta.write().await = Some(crate::services::rebalance::RebalanceMeta {
9447 pool_stats: vec![crate::services::rebalance::RebalanceStats::default(), rebalancing],
9448 ..Default::default()
9449 });
9450 let error = store
9451 .delete_object(
9452 bucket,
9453 object,
9454 ObjectOptions {
9455 versioned: true,
9456 ..Default::default()
9457 },
9458 )
9459 .await;
9460 *store.rebalance_meta.write().await = None;
9461 assert_eq!(
9462 error.expect_err("rebalance must not hide the owner during marker lookup"),
9463 StorageError::SlowDown
9464 );
9465
9466 let quorum_bucket = RUSTFS_META_BUCKET;
9469 store.pools[1]
9470 .put_object(
9471 quorum_bucket,
9472 object,
9473 &mut PutObjReader::from_vec(vec![1; 4097]),
9474 &ObjectOptions {
9475 versioned: true,
9476 ..Default::default()
9477 },
9478 )
9479 .await
9480 .expect("seed the object-quorum fixture");
9481 for pool in &store.pools {
9482 pool.put_object(
9483 quorum_bucket,
9484 "split.bin",
9485 &mut PutObjReader::from_vec(vec![2; 4097]),
9486 &ObjectOptions {
9487 versioned: true,
9488 ..Default::default()
9489 },
9490 )
9491 .await
9492 .expect("seed another key with history in both pools");
9493 }
9494
9495 let owner = &store.pools[1].disk_set[0];
9496 let healthy = owner.disks.read().await.clone();
9497 for disk in owner.disks.write().await.iter_mut().skip(1) {
9498 *disk = None;
9499 }
9500 let error = store
9501 .delete_object(
9502 quorum_bucket,
9503 object,
9504 ObjectOptions {
9505 versioned: true,
9506 ..Default::default()
9507 },
9508 )
9509 .await;
9510 let split_error = store
9511 .delete_object(
9512 quorum_bucket,
9513 "split.bin",
9514 ObjectOptions {
9515 versioned: true,
9516 ..Default::default()
9517 },
9518 )
9519 .await;
9520 *owner.disks.write().await = healthy;
9521 assert_eq!(
9522 error.expect_err("subquorum owner must not become a new-pool marker"),
9523 StorageError::ErasureWriteQuorum
9524 );
9525 assert_eq!(
9526 split_error.expect_err("a readable older pool does not prove the global current version"),
9527 StorageError::ErasureWriteQuorum
9528 );
9529 let split = store.pools[0]
9530 .clone()
9531 .inner_list_object_versions(quorum_bucket, "split.bin", None, None, None, 10)
9532 .await
9533 .expect("inspect the readable older pool");
9534 assert_eq!(split.objects.len(), 1);
9535 assert!(!split.objects[0].delete_marker);
9536 let other = store.pools[0]
9537 .clone()
9538 .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9539 .await
9540 .expect("inspect the other pool");
9541 assert!(other.objects.is_empty());
9542 let history = store.pools[1]
9543 .clone()
9544 .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9545 .await
9546 .expect("inspect the restored owner");
9547 assert_eq!(history.objects.len(), 1);
9548 assert!(!history.objects[0].delete_marker);
9549 }
9550
9551 #[tokio::test]
9552 async fn multipool_suspended_and_batch_markers_keep_the_null_slot_on_the_owner() {
9553 let bucket = "multipool-suspended-markers";
9554 let object = "history.bin";
9555 let (_dirs, store) = multipool_version_test_store(bucket).await;
9556 let original = store.pools[1]
9557 .put_object(
9558 bucket,
9559 object,
9560 &mut PutObjReader::from_vec(vec![1; 4097]),
9561 &ObjectOptions {
9562 versioned: true,
9563 ..Default::default()
9564 },
9565 )
9566 .await
9567 .expect("seed the UUID history in pool 1");
9568 store
9569 .update_bucket_metadata_config(
9570 bucket,
9571 crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
9572 b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
9573 )
9574 .await
9575 .expect("persist suspended bucket versioning");
9576 let null = store
9577 .put_object(
9578 bucket,
9579 object,
9580 &mut PutObjReader::from_vec(vec![2; 4097]),
9581 &ObjectOptions {
9582 version_suspended: true,
9583 ..Default::default()
9584 },
9585 )
9586 .await
9587 .expect("write the null version beside the history");
9588 assert!(null.version_id.is_none_or(|id| id.is_nil()));
9589 for _ in 0..2 {
9590 let marker = store
9591 .delete_object(
9592 bucket,
9593 object,
9594 ObjectOptions {
9595 version_suspended: true,
9596 ..Default::default()
9597 },
9598 )
9599 .await
9600 .expect("replace the null slot with a marker");
9601 assert!(marker.delete_marker);
9602 assert!(marker.version_id.is_none_or(|id| id.is_nil()));
9603 }
9604 let (deleted, errors) = store
9605 .delete_objects(
9606 bucket,
9607 vec![ObjectToDelete {
9608 object_name: object.to_string(),
9609 ..Default::default()
9610 }],
9611 ObjectOptions::default(),
9612 )
9613 .await;
9614 assert!(errors.iter().all(Option::is_none), "batch DELETE should succeed: {errors:?}");
9615 assert_eq!(deleted.len(), 1);
9616 assert!(deleted[0].delete_marker);
9617 let versions = store.pools[1]
9618 .clone()
9619 .inner_list_object_versions(bucket, object, None, None, None, 10)
9620 .await
9621 .expect("inspect the owner after batch DELETE");
9622 assert_eq!(versions.objects.len(), 2, "only one null marker and the UUID history remain");
9623 assert!(versions.objects[0].delete_marker && versions.objects[0].is_latest);
9624 assert_eq!(versions.objects[1].version_id, original.version_id);
9625 let other = store.pools[0]
9626 .clone()
9627 .inner_list_object_versions(bucket, object, None, None, None, 10)
9628 .await
9629 .expect("inspect the unused pool");
9630 assert!(other.objects.is_empty());
9631 }
9632
9633 async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
9634 assert_pool_writer_is_blocked(store, 0, bucket, object).await;
9635 }
9636
9637 async fn assert_pool_writer_is_blocked(store: &ECStore, pool_idx: usize, bucket: &str, object: &str) {
9638 let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9639 let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9640 let err = lock
9641 .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
9642 .await
9643 .expect_err("prepared read lock should block the writer");
9644 assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
9645 }
9646
9647 async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
9648 acquire_pool_writer(store, 0, bucket, object).await
9649 }
9650
9651 async fn acquire_pool_writer(
9652 store: &ECStore,
9653 pool_idx: usize,
9654 bucket: &str,
9655 object: &str,
9656 ) -> rustfs_lock::NamespaceLockGuard {
9657 let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9658 let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9659 lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
9660 .await
9661 .expect("prepared read lock should have been released")
9662 }
9663
9664 #[tokio::test]
9665 #[serial_test::serial(body_cache_hook)]
9666 async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
9667 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9668 let store = new_prepared_reader_test_store(&[set_disks]).await;
9669 let bucket = "prepared-reader-hook-miss";
9670 let object = "object.bin";
9671 let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
9672 let opts = ObjectOptions {
9673 no_lock: true,
9674 ..Default::default()
9675 };
9676
9677 store.pools[0]
9678 .make_bucket(bucket, &MakeBucketOptions::default())
9679 .await
9680 .expect("bucket should be created");
9681 let mut put_reader = PutObjReader::from_vec(payload.clone());
9682 store.pools[0]
9683 .put_object(bucket, object, &mut put_reader, &opts)
9684 .await
9685 .expect("object should be written");
9686
9687 clear_get_object_body_cache_hook();
9688 let hook = Arc::new(CountingMissHook {
9689 calls: AtomicUsize::new(0),
9690 });
9691 register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
9692 let _hook_guard = BodyCacheHookGuard;
9693
9694 let prepared = store
9695 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
9696 .await
9697 .expect("prepared reader metadata should resolve");
9698 assert!(matches!(
9699 lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
9700 GetObjectBodyCacheHookLookup::Miss
9701 ));
9702 assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");
9703
9704 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9705 assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
9706 let mut restored = Vec::new();
9707 reader
9708 .stream
9709 .read_to_end(&mut restored)
9710 .await
9711 .expect("prepared body should stream");
9712
9713 assert_eq!(restored, payload);
9714 assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
9715 }
9716
9717 #[tokio::test]
9718 #[serial_test::serial(body_cache_hook)]
9719 async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
9720 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
9721 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9722 let store = new_prepared_reader_test_store(&[set_disks]).await;
9723 let bucket = "prepared-reader-lock-lifetime";
9724 let object = "object.bin";
9725 let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
9726 let put_opts = ObjectOptions {
9727 no_lock: true,
9728 ..Default::default()
9729 };
9730
9731 store.pools[0]
9732 .make_bucket(bucket, &MakeBucketOptions::default())
9733 .await
9734 .expect("bucket should be created");
9735 let mut put_reader = PutObjReader::from_vec(payload.clone());
9736 store.pools[0]
9737 .put_object(bucket, object, &mut put_reader, &put_opts)
9738 .await
9739 .expect("object should be written");
9740
9741 let prepared = store
9742 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9743 .await
9744 .expect("prepared reader metadata should resolve");
9745 assert!(prepared.read_lock_guard.is_some());
9746 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9747
9748 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9749 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9750 let mut restored = Vec::new();
9751 reader
9752 .stream
9753 .read_to_end(&mut restored)
9754 .await
9755 .expect("prepared body should stream");
9756 assert_eq!(restored, payload);
9757 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9758
9759 let prepared = store
9760 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9761 .await
9762 .expect("second prepared reader metadata should resolve");
9763 let reader = prepared.into_reader().await.expect("second prepared body reader should open");
9764 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9765 drop(reader);
9766 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9767 })
9768 .await;
9769 }
9770
9771 #[tokio::test]
9772 #[serial_test::serial(body_cache_hook)]
9773 async fn select_snapshot_holds_namespace_lock_independent_of_get_optimization() {
9774 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
9775 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9776 let store = new_prepared_reader_test_store(&[set_disks]).await;
9777 let bucket = "select-snapshot-lock-lifetime";
9778 let object = "object.bin";
9779 let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
9780 let put_opts = ObjectOptions {
9781 no_lock: true,
9782 ..Default::default()
9783 };
9784
9785 store.pools[0]
9786 .make_bucket(bucket, &MakeBucketOptions::default())
9787 .await
9788 .expect("bucket should be created");
9789 let mut put_reader = PutObjReader::from_vec(payload.clone());
9790 store.pools[0]
9791 .put_object(bucket, object, &mut put_reader, &put_opts)
9792 .await
9793 .expect("object should be written");
9794
9795 use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
9796 let mut request_headers = HeaderMap::new();
9797 request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
9798 request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
9799 request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
9800 request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
9801 let snapshot = store
9802 .prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
9803 .await
9804 .expect("SelectObjectContent snapshot should be prepared");
9805 assert_eq!(snapshot.headers.len(), 3);
9806 assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
9807 assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
9808 assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
9809 assert!(snapshot.headers.get("authorization").is_none());
9810 assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
9811 assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
9812 assert_eq!(
9813 snapshot.logical_size(),
9814 u64::try_from(payload.len()).expect("test payload length should fit in u64")
9815 );
9816 assert_eq!(
9817 snapshot.object_info().size,
9818 i64::try_from(payload.len()).expect("test payload length should fit in i64")
9819 );
9820 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9821
9822 let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
9823 let mut restored = Vec::new();
9824 reader
9825 .stream
9826 .read_to_end(&mut restored)
9827 .await
9828 .expect("snapshot body should stream");
9829 assert_eq!(restored, payload);
9830 drop(reader);
9831 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9832
9833 drop(snapshot);
9834 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9835 })
9836 .await;
9837 }
9838
9839 #[tokio::test]
9840 async fn select_snapshot_identity_compares_encoded_directory_object_key() {
9841 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9842 let store = new_prepared_reader_test_store(&[set_disks]).await;
9843 let bucket = "select-snapshot-directory-identity";
9844 let source_object = "source.bin";
9845
9846 store.pools[0]
9847 .make_bucket(bucket, &MakeBucketOptions::default())
9848 .await
9849 .expect("bucket should be created");
9850 let mut put_reader = PutObjReader::from_vec(b"source".to_vec());
9851 store.pools[0]
9852 .put_object(
9853 bucket,
9854 source_object,
9855 &mut put_reader,
9856 &ObjectOptions {
9857 no_lock: true,
9858 ..Default::default()
9859 },
9860 )
9861 .await
9862 .expect("source object should be written");
9863
9864 let mut snapshot = store
9865 .prepare_select_object_snapshot(bucket, source_object, &HeaderMap::new(), &ObjectOptions::default())
9866 .await
9867 .expect("source object snapshot should be prepared");
9868 snapshot.object = encode_dir_object("directory/");
9869
9870 assert!(snapshot.is_for(bucket, "directory/"));
9871 assert!(!snapshot.is_for(bucket, "different/"));
9872 }
9873
9874 #[tokio::test]
9875 #[serial_test::serial(body_cache_hook)]
9876 async fn select_snapshot_reuses_initial_metadata_fanout_for_first_reader() {
9877 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9878 let store = new_prepared_reader_test_store(&[set_disks]).await;
9879 let bucket = "select-snapshot-initial-metadata";
9880 let object = "initial-metadata.bin";
9881 let payload = b"select snapshot initial metadata".repeat(4_000);
9882 let no_lock_opts = ObjectOptions {
9883 no_lock: true,
9884 ..Default::default()
9885 };
9886
9887 store.pools[0]
9888 .make_bucket(bucket, &MakeBucketOptions::default())
9889 .await
9890 .expect("bucket should be created");
9891 let mut put_reader = PutObjReader::from_vec(payload.clone());
9892 store.pools[0]
9893 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
9894 .await
9895 .expect("object should be written");
9896
9897 let calls = disk_call_counters::observe(object);
9898 let snapshot = store
9899 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
9900 .await
9901 .expect("snapshot should prepare metadata");
9902 assert_eq!(
9903 calls.total(disk_call_counters::KIND_READ_VERSION),
9904 4,
9905 "snapshot preparation should fan out metadata once"
9906 );
9907
9908 let mut reader = snapshot.open_reader(None).await.expect("first snapshot reader should open");
9909 assert_eq!(
9910 calls.total(disk_call_counters::KIND_READ_VERSION),
9911 4,
9912 "first reader should consume the metadata captured during preparation"
9913 );
9914 let mut restored = Vec::new();
9915 reader
9916 .stream
9917 .read_to_end(&mut restored)
9918 .await
9919 .expect("snapshot body should stream");
9920 assert_eq!(restored, payload);
9921 }
9922
9923 #[tokio::test]
9924 #[serial_test::serial(body_cache_hook)]
9925 async fn select_snapshot_rejects_identity_change_before_second_reader() {
9926 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9927 let store = new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await;
9928 let bucket = "select-snapshot-identity-change";
9929 let object = "identity-change.bin";
9930 let no_lock_opts = ObjectOptions {
9931 no_lock: true,
9932 ..Default::default()
9933 };
9934
9935 set_disks
9936 .make_bucket(bucket, &MakeBucketOptions::default())
9937 .await
9938 .expect("bucket should be created");
9939 let mut initial_reader = PutObjReader::from_vec(b"first generation".to_vec());
9940 set_disks
9941 .put_object(bucket, object, &mut initial_reader, &no_lock_opts)
9942 .await
9943 .expect("initial generation should be written");
9944 let mut snapshot = store
9945 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
9946 .await
9947 .expect("snapshot should be prepared");
9948 drop(
9949 snapshot
9950 .open_reader(None)
9951 .await
9952 .expect("first snapshot reader should consume captured metadata"),
9953 );
9954 snapshot.opts.metadata_cache_safe = false;
9955
9956 let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
9957 set_disks
9958 .put_object(bucket, object, &mut replacement_reader, &no_lock_opts)
9959 .await
9960 .expect("test-only no-lock write should replace the object");
9961 let error = match snapshot.open_reader(None).await {
9962 Ok(_) => panic!("a later reader must reject the replacement generation"),
9963 Err(error) => error,
9964 };
9965 assert!(matches!(
9966 error,
9967 SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged)
9968 ));
9969 }
9970
9971 #[tokio::test]
9972 #[serial_test::serial(body_cache_hook)]
9973 async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() {
9974 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
9975 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
9976 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
9977 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
9978 let bucket = "select-snapshot-latest-delete-marker";
9979 let object = "versioned-object.bin";
9980 let versioned_opts = ObjectOptions {
9981 no_lock: true,
9982 versioned: true,
9983 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
9984 ..Default::default()
9985 };
9986
9987 for set_disks in [&first_set, &second_set] {
9988 set_disks
9989 .make_bucket(bucket, &MakeBucketOptions::default())
9990 .await
9991 .expect("bucket should be created");
9992 }
9993 let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
9994 first_set
9995 .put_object(bucket, object, &mut older_reader, &versioned_opts)
9996 .await
9997 .expect("older versioned object should be written");
9998 let mut put_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
9999 second_set
10000 .put_object(bucket, object, &mut put_reader, &versioned_opts)
10001 .await
10002 .expect("versioned object should be written");
10003 let marker = second_set
10004 .delete_object(bucket, object, versioned_opts.clone())
10005 .await
10006 .expect("versioned delete should create a marker");
10007 assert!(marker.delete_marker);
10008 assert!(marker.version_id.is_some_and(|version_id| !version_id.is_nil()));
10009
10010 let error = store
10011 .prepare_select_object_snapshot(
10012 bucket,
10013 object,
10014 &HeaderMap::new(),
10015 &ObjectOptions {
10016 versioned: true,
10017 ..Default::default()
10018 },
10019 )
10020 .await
10021 .expect_err("latest delete marker should hide the prior object generation");
10022 assert!(matches!(
10023 error,
10024 PrepareSelectObjectSnapshotError::Storage(ref error) if is_err_object_not_found(error)
10025 ));
10026 }
10027
10028 #[tokio::test]
10029 #[serial_test::serial(storage_class_env)]
10030 async fn select_snapshot_blocks_store_delete_for_object_in_nonzero_set() {
10031 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10032 let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
10033 ctx.update_erasure_type(SetupType::DistErasure).await;
10034 assert!(
10035 sets.disk_set[0]
10036 .lockers
10037 .iter()
10038 .all(|first| sets.disk_set[1].lockers.iter().all(|second| !Arc::ptr_eq(first, second)))
10039 );
10040 let pool_config = sets.endpoints.clone();
10041 let store = Arc::new(new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx));
10042 let bucket = RUSTFS_META_BUCKET;
10043 let object = (0..1_000)
10044 .map(|index| format!("nonzero-set-{index}.bin"))
10045 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
10046 .expect("a key should hash to the second set");
10047 let mut put_reader = PutObjReader::from_vec(b"stable snapshot body".to_vec());
10048 sets.put_object(
10049 bucket,
10050 &object,
10051 &mut put_reader,
10052 &ObjectOptions {
10053 no_lock: true,
10054 ..Default::default()
10055 },
10056 )
10057 .await
10058 .expect("object should be written to the second set");
10059 assert!(Arc::ptr_eq(&sets.get_disks_by_key(&object), &sets.disk_set[1]));
10060
10061 let snapshot = store
10062 .prepare_select_object_snapshot(bucket, &object, &HeaderMap::new(), &ObjectOptions::default())
10063 .await
10064 .expect("snapshot should acquire both lock domains");
10065 let hashed_set_writer = rustfs_lock::NamespaceLock::with_clients_and_quorum(
10066 "select-nonzero-set-writer".to_string(),
10067 sets.disk_set[1].lockers.clone(),
10068 2,
10069 );
10070 let writer_error = hashed_set_writer
10071 .get_write_lock(
10072 rustfs_lock::ObjectKey::new(bucket, object.as_str()),
10073 "competing-hashed-set-writer",
10074 Duration::from_millis(50),
10075 )
10076 .await
10077 .expect_err("Select must hold the nonzero hashed-set read lock");
10078 assert!(matches!(writer_error, rustfs_lock::LockError::Timeout { .. }));
10079 let barrier = DeleteAfterObjectLockSnapshotBarrier::install(bucket);
10080 let delete_store = Arc::clone(&store);
10081 let delete_object = object.clone();
10082 let mut delete = tokio::spawn(async move {
10083 delete_store
10084 .delete_object(bucket, &delete_object, ObjectOptions::default())
10085 .await
10086 });
10087 barrier.wait_until_paused().await;
10088 barrier.release();
10089 assert!(
10090 tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(),
10091 "store DELETE must wait for the fixed-domain Select read lock"
10092 );
10093
10094 drop(snapshot);
10095 let deleted = tokio::time::timeout(Duration::from_secs(10), delete)
10096 .await
10097 .expect("DELETE should resume after the snapshot is dropped")
10098 .expect("DELETE task should not panic")
10099 .expect("DELETE should complete");
10100 assert_eq!(deleted.name, object);
10101 }
10102
10103 #[tokio::test]
10104 #[serial_test::serial]
10105 async fn select_snapshot_fails_closed_when_local_locking_is_disabled() {
10106 let ctx = Arc::new(crate::runtime::instance::InstanceContext::with_lock_manager_for_test(Arc::new(
10107 rustfs_lock::GlobalLockManager::Disabled(rustfs_lock::DisabledLockManager::new()),
10108 )));
10109 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10110 let store = new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await;
10111 let bucket = "select-snapshot-lock-disabled";
10112 let object = "object.bin";
10113 let mut put_reader = PutObjReader::from_vec(b"payload".to_vec());
10114 let no_lock_opts = ObjectOptions {
10115 no_lock: true,
10116 ..Default::default()
10117 };
10118
10119 store.pools[0]
10120 .make_bucket(bucket, &MakeBucketOptions::default())
10121 .await
10122 .expect("bucket should be created");
10123 store.pools[0]
10124 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10125 .await
10126 .expect("object should be written without namespace locking");
10127
10128 let error = store
10129 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10130 .await
10131 .expect_err("SelectObjectContent must reject a disabled lock manager");
10132 assert!(matches!(
10133 error,
10134 PrepareSelectObjectSnapshotError::Consistency(SnapshotConsistencyError::LockingDisabled)
10135 ));
10136 }
10137
10138 #[tokio::test]
10139 #[serial_test::serial(body_cache_hook)]
10140 async fn prepared_object_info_releases_namespace_lock_immediately() {
10141 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10142 let store = new_prepared_reader_test_store(&[set_disks]).await;
10143 let bucket = "prepared-object-info-lock-release";
10144 let object = "object.bin";
10145 let payload = b"prepared-object-info-lock-release".to_vec();
10146 let put_opts = ObjectOptions {
10147 no_lock: true,
10148 ..Default::default()
10149 };
10150
10151 store.pools[0]
10152 .make_bucket(bucket, &MakeBucketOptions::default())
10153 .await
10154 .expect("bucket should be created");
10155 let mut put_reader = PutObjReader::from_vec(payload.clone());
10156 store.pools[0]
10157 .put_object(bucket, object, &mut put_reader, &put_opts)
10158 .await
10159 .expect("object should be written");
10160
10161 let prepared = store
10162 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
10163 .await
10164 .expect("prepared reader metadata should resolve");
10165 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10166 assert_eq!(prepared.into_object_info().size, payload.len() as i64);
10167
10168 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10169 }
10170
10171 #[tokio::test]
10172 #[serial_test::serial(body_cache_hook)]
10173 async fn prepared_reader_reuses_metadata_across_three_pools() {
10174 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10175 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10176 let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10177 let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10178 let bucket = "prepared-reader-three-pools";
10179 let object = "object.bin";
10180 let payload = b"prepared-reader-three-pool-payload-".repeat(40_000);
10181 let opts = ObjectOptions {
10182 no_lock: true,
10183 ..Default::default()
10184 };
10185
10186 for pool in &store.pools {
10187 pool.make_bucket(bucket, &MakeBucketOptions::default())
10188 .await
10189 .expect("bucket should be created in each pool");
10190 }
10191 let mut put_reader = PutObjReader::from_vec(payload.clone());
10192 store.pools[2]
10193 .put_object(bucket, object, &mut put_reader, &opts)
10194 .await
10195 .expect("object should be written only to the third pool");
10196
10197 let calls = disk_call_counters::observe(object);
10198 let prepared = store
10199 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10200 .await
10201 .expect("prepared reader should resolve the third-pool object");
10202 assert_eq!(prepared.object_info().size, payload.len() as i64);
10203 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10204 assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10205 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
10206 assert_eq!(
10207 calls.total(disk_call_counters::KIND_READ_VERSION),
10208 metadata_calls,
10209 "the selected pool must reuse its prepared metadata"
10210 );
10211 let mut restored = Vec::new();
10212 reader
10213 .stream
10214 .read_to_end(&mut restored)
10215 .await
10216 .expect("prepared body should stream");
10217 assert_eq!(restored, payload);
10218 }
10219
10220 #[tokio::test]
10221 #[serial_test::serial(body_cache_hook)]
10222 async fn select_snapshot_reuses_metadata_across_three_pools() {
10223 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10224 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10225 let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10226 let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10227 let bucket = "select-snapshot-three-pools";
10228 let object = "object.bin";
10229 let payload = b"select-snapshot-three-pool-payload-".repeat(40_000);
10230 let write_opts = ObjectOptions {
10231 no_lock: true,
10232 ..Default::default()
10233 };
10234
10235 for pool in &store.pools {
10236 pool.make_bucket(bucket, &MakeBucketOptions::default())
10237 .await
10238 .expect("bucket should be created in each pool");
10239 }
10240 let mut put_reader = PutObjReader::from_vec(payload.clone());
10241 store.pools[2]
10242 .put_object(bucket, object, &mut put_reader, &write_opts)
10243 .await
10244 .expect("object should be written only to the third pool");
10245
10246 let calls = disk_call_counters::observe(object);
10247 let snapshot = store
10248 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10249 .await
10250 .expect("SelectObjectContent snapshot should resolve the third-pool object");
10251 assert_eq!(snapshot.object_info().size, payload.len() as i64);
10252 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10253 assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10254
10255 let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open");
10256 assert_eq!(
10257 calls.total(disk_call_counters::KIND_READ_VERSION),
10258 metadata_calls,
10259 "SelectObjectContent must consume the prepared winner without a second fanout"
10260 );
10261 let mut restored = Vec::new();
10262 reader
10263 .stream
10264 .read_to_end(&mut restored)
10265 .await
10266 .expect("snapshot body should stream");
10267 assert_eq!(restored, payload);
10268 }
10269
10270 #[tokio::test]
10271 #[serial_test::serial(body_cache_hook)]
10272 async fn legacy_reader_reuses_selected_pool_metadata() {
10273 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10274 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10275 let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10276 let bucket = "legacy-reader-second-pool";
10277 let object = "object.bin";
10278 let payload = b"legacy-reader-second-pool-payload-".repeat(40_000);
10279 let opts = ObjectOptions {
10280 no_lock: true,
10281 ..Default::default()
10282 };
10283
10284 for pool in &store.pools {
10285 pool.make_bucket(bucket, &MakeBucketOptions::default())
10286 .await
10287 .expect("bucket should be created in each pool");
10288 }
10289 let mut put_reader = PutObjReader::from_vec(payload.clone());
10290 store.pools[1]
10291 .put_object(bucket, object, &mut put_reader, &opts)
10292 .await
10293 .expect("object should be written only to the second pool");
10294
10295 clear_get_object_body_cache_hook();
10296 let hook = Arc::new(CountingMissHook {
10297 calls: AtomicUsize::new(0),
10298 });
10299 register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
10300 let _hook_guard = BodyCacheHookGuard;
10301
10302 let calls = disk_call_counters::observe(object);
10303 let mut reader = store
10304 .handle_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10305 .await
10306 .expect("legacy reader should resolve the second-pool object");
10307 assert_eq!(
10308 hook.calls.load(Ordering::Relaxed),
10309 1,
10310 "legacy reader must probe the body cache exactly once"
10311 );
10312 assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
10313 assert!(
10314 calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10315 "legacy reader must fan out each 4-disk pool at most once"
10316 );
10317 let mut restored = Vec::new();
10318 reader
10319 .stream
10320 .read_to_end(&mut restored)
10321 .await
10322 .expect("legacy reader body should stream");
10323 assert_eq!(restored, payload);
10324 }
10325
10326 fn prepared_pool_test_status(id: usize, suspended: bool) -> PoolStatus {
10327 PoolStatus {
10328 id,
10329 cmd_line: format!("prepared-pool-{id}"),
10330 last_update: OffsetDateTime::now_utc(),
10331 decommission: suspended.then(|| PoolDecommissionInfo {
10332 start_time: Some(OffsetDateTime::now_utc()),
10333 ..Default::default()
10334 }),
10335 }
10336 }
10337
10338 #[tokio::test]
10339 #[serial_test::serial(body_cache_hook)]
10340 async fn prepared_reader_refetches_when_final_pool_state_changes_winner() {
10341 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10342 let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10343 let bucket = "prepared-reader-pool-state-fallback";
10344 let object = "object.bin";
10345 let payload = b"pool-state fallback payload".repeat(8_000);
10346 let opts = ObjectOptions {
10347 no_lock: true,
10348 ..Default::default()
10349 };
10350
10351 set_disks
10352 .make_bucket(bucket, &MakeBucketOptions::default())
10353 .await
10354 .expect("bucket should be created");
10355 let mut put_reader = PutObjReader::from_vec(payload.clone());
10356 set_disks
10357 .put_object(bucket, object, &mut put_reader, &opts)
10358 .await
10359 .expect("shared object should be written");
10360
10361 let calls = disk_call_counters::observe(object);
10362 let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, false);
10363 let read_store = Arc::clone(&store);
10364 let read_opts = opts.clone();
10365 let read = tokio::spawn(async move {
10366 read_store
10367 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10368 .await
10369 });
10370 barrier.wait_after_fanout().await;
10371 *store.pool_meta.write().await = PoolMeta {
10372 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10373 ..Default::default()
10374 };
10375 barrier.release_after_fanout();
10376
10377 let prepared = read
10378 .await
10379 .expect("prepared read task should not panic")
10380 .expect("final active pool should be refetched");
10381 assert!(Arc::ptr_eq(&prepared.pool, &store.pools[0]));
10382 assert_eq!(
10383 calls.total(disk_call_counters::KIND_READ_VERSION),
10384 12,
10385 "two initial 4-disk fanouts plus one fallback refetch are required"
10386 );
10387 let mut reader = prepared.into_reader().await.expect("fallback body reader should open");
10388 let mut restored = Vec::new();
10389 reader
10390 .stream
10391 .read_to_end(&mut restored)
10392 .await
10393 .expect("fallback body should stream");
10394 assert_eq!(restored, payload);
10395 }
10396
10397 #[tokio::test]
10398 #[serial_test::serial(body_cache_hook)]
10399 async fn prepared_reader_fallback_rejects_generation_change_before_refetch() {
10400 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10401 let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10402 let bucket = "prepared-reader-pool-state-generation-change";
10403 let object = "object.bin";
10404 let opts = ObjectOptions {
10405 no_lock: true,
10406 ..Default::default()
10407 };
10408
10409 set_disks
10410 .make_bucket(bucket, &MakeBucketOptions::default())
10411 .await
10412 .expect("bucket should be created");
10413 let mut initial_reader = PutObjReader::from_vec(b"initial generation".to_vec());
10414 set_disks
10415 .put_object(bucket, object, &mut initial_reader, &opts)
10416 .await
10417 .expect("initial object should be written");
10418
10419 let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, true);
10420 let read_store = Arc::clone(&store);
10421 let read_opts = opts.clone();
10422 let read = tokio::spawn(async move {
10423 read_store
10424 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10425 .await
10426 });
10427 barrier.wait_after_fanout().await;
10428 *store.pool_meta.write().await = PoolMeta {
10429 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10430 ..Default::default()
10431 };
10432 barrier.release_after_fanout();
10433 barrier.wait_before_refetch().await;
10434
10435 let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10436 set_disks
10437 .put_object(bucket, object, &mut replacement_reader, &opts)
10438 .await
10439 .expect("replacement generation should be written before fallback refetch");
10440 barrier.release_before_refetch();
10441
10442 let error = match read.await.expect("prepared read task should not panic") {
10443 Ok(_) => panic!("changed fallback generation must not be accepted"),
10444 Err(error) => error,
10445 };
10446 assert_eq!(error, Error::ErasureReadQuorum);
10447 }
10448
10449 #[tokio::test]
10450 #[serial_test::serial(body_cache_hook)]
10451 async fn prepared_reader_rejects_latest_delete_marker_without_refetching_metadata() {
10452 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10453 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10454 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10455 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10456 let bucket = "prepared-reader-latest-delete-marker";
10457 let object = "versioned-object.bin";
10458 let versioned_opts = ObjectOptions {
10459 no_lock: true,
10460 versioned: true,
10461 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10462 ..Default::default()
10463 };
10464
10465 for set_disks in [&first_set, &second_set] {
10466 set_disks
10467 .make_bucket(bucket, &MakeBucketOptions::default())
10468 .await
10469 .expect("bucket should be created");
10470 }
10471 let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10472 first_set
10473 .put_object(bucket, object, &mut older_reader, &versioned_opts)
10474 .await
10475 .expect("older object should be written");
10476 let mut hidden_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10477 second_set
10478 .put_object(bucket, object, &mut hidden_reader, &versioned_opts)
10479 .await
10480 .expect("newer object should be written");
10481 let marker = second_set
10482 .delete_object(bucket, object, versioned_opts.clone())
10483 .await
10484 .expect("delete marker should be committed");
10485 assert!(marker.delete_marker);
10486
10487 let calls = disk_call_counters::observe(object);
10488 let error = match store
10489 .prepare_get_object_reader(
10490 bucket,
10491 object,
10492 None,
10493 HeaderMap::new(),
10494 &ObjectOptions {
10495 no_lock: true,
10496 versioned: true,
10497 ..Default::default()
10498 },
10499 )
10500 .await
10501 {
10502 Ok(_) => panic!("latest delete marker should hide the older live object"),
10503 Err(error) => error,
10504 };
10505
10506 assert!(is_err_object_not_found(&error));
10507 assert!(
10508 calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10509 "delete-marker resolution must fan out each pool at most once"
10510 );
10511 }
10512
10513 #[tokio::test]
10514 #[serial_test::serial(body_cache_hook)]
10515 async fn prepared_reader_explicit_version_reuses_the_matching_pool_metadata() {
10516 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10517 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10518 let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await;
10519 let bucket = "prepared-reader-explicit-version";
10520 let object = "versioned-object.bin";
10521 let payload = b"explicit version from first pool".repeat(8_000);
10522 let versioned_opts = ObjectOptions {
10523 no_lock: true,
10524 versioned: true,
10525 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10526 ..Default::default()
10527 };
10528
10529 for set_disks in [&first_set, &second_set] {
10530 set_disks
10531 .make_bucket(bucket, &MakeBucketOptions::default())
10532 .await
10533 .expect("bucket should be created");
10534 }
10535 let mut first_reader = PutObjReader::from_vec(payload.clone());
10536 let first = first_set
10537 .put_object(bucket, object, &mut first_reader, &versioned_opts)
10538 .await
10539 .expect("requested version should be written to the first pool");
10540 let mut second_reader = PutObjReader::from_vec(b"different pool version".to_vec());
10541 second_set
10542 .put_object(bucket, object, &mut second_reader, &versioned_opts)
10543 .await
10544 .expect("a different version should be written to the second pool");
10545
10546 let requested_version = first
10547 .version_id
10548 .expect("versioned PUT should return a version id")
10549 .to_string();
10550 let read_opts = ObjectOptions {
10551 no_lock: true,
10552 versioned: true,
10553 version_id: Some(requested_version),
10554 ..Default::default()
10555 };
10556 let calls = disk_call_counters::observe(object);
10557 let prepared = store
10558 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10559 .await
10560 .expect("explicit version should resolve from the matching pool");
10561 assert_eq!(prepared.object_info().version_id, first.version_id);
10562 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10563 assert!(metadata_calls <= 8, "explicit-version lookup must fan out each pool at most once");
10564
10565 let mut reader = prepared
10566 .into_reader()
10567 .await
10568 .expect("prepared explicit-version body should open");
10569 assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), metadata_calls);
10570 let mut restored = Vec::new();
10571 reader
10572 .stream
10573 .read_to_end(&mut restored)
10574 .await
10575 .expect("explicit-version body should stream");
10576 assert_eq!(restored, payload);
10577 }
10578
10579 #[tokio::test]
10580 #[serial_test::serial]
10581 async fn select_snapshot_locks_the_hashed_set_in_every_pool() {
10582 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10583 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10584 let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10585 let bucket = "select-snapshot-all-pools";
10586 let object = "object.bin";
10587 let payload = b"second-pool-snapshot".to_vec();
10588 let no_lock_opts = ObjectOptions {
10589 no_lock: true,
10590 ..Default::default()
10591 };
10592
10593 for pool in &store.pools {
10594 pool.make_bucket(bucket, &MakeBucketOptions::default())
10595 .await
10596 .expect("bucket should be created in each pool");
10597 }
10598 let mut put_reader = PutObjReader::from_vec(payload.clone());
10599 store.pools[1]
10600 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10601 .await
10602 .expect("object should be written only to the second pool");
10603
10604 let snapshot = store
10605 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10606 .await
10607 .expect("snapshot should resolve the second-pool object");
10608 assert_pool_writer_is_blocked(&store, 0, bucket, object).await;
10609 assert_pool_writer_is_blocked(&store, 1, bucket, object).await;
10610
10611 let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10612 let mut restored = Vec::new();
10613 reader
10614 .stream
10615 .read_to_end(&mut restored)
10616 .await
10617 .expect("snapshot body should stream");
10618 assert_eq!(restored, payload);
10619 drop(reader);
10620 drop(snapshot);
10621
10622 drop(acquire_pool_writer(&store, 0, bucket, object).await);
10623 drop(acquire_pool_writer(&store, 1, bucket, object).await);
10624 }
10625
10626 #[tokio::test]
10630 async fn instance_context_flows_through_object_graph() {
10631 let store = new_read_lock_test_store().await;
10632
10633 let sets = store.pools.first().expect("test store has one pool");
10634 assert!(
10635 std::sync::Arc::ptr_eq(&store.ctx, sets.instance_ctx()),
10636 "Sets must carry the store's instance context"
10637 );
10638
10639 let set_disks = sets.disk_set.first().expect("pool has one set");
10640 assert!(
10641 std::sync::Arc::ptr_eq(sets.instance_ctx(), set_disks.instance_ctx()),
10642 "SetDisks must carry the Sets' instance context"
10643 );
10644 }
10645
10646 #[tokio::test]
10651 async fn set_disks_lock_manager_comes_from_instance_context() {
10652 let store = new_read_lock_test_store().await;
10653 let set_disks = store.pools[0].disk_set.first().expect("pool has one set");
10654
10655 assert!(
10656 std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &set_disks.instance_ctx().lock_manager()),
10657 "SetDisks lock manager must be sourced from its instance context"
10658 );
10659 assert!(
10660 std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &rustfs_lock::get_global_lock_manager()),
10661 "single-instance lock manager must alias the process singleton"
10662 );
10663 }
10664
10665 #[tokio::test]
10666 #[serial_test::serial]
10667 async fn delete_objects_write_locks_cover_each_unique_object() {
10668 let store = new_read_lock_test_store().await;
10669 let objects = vec![
10670 ObjectToDelete {
10671 object_name: "beta".to_string(),
10672 ..Default::default()
10673 },
10674 ObjectToDelete {
10675 object_name: "alpha".to_string(),
10676 ..Default::default()
10677 },
10678 ObjectToDelete {
10679 object_name: "beta".to_string(),
10680 ..Default::default()
10681 },
10682 ];
10683 let mut opts = ObjectOptions::default();
10684
10685 let guards = store
10686 .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
10687 .await
10688 .expect("delete object locks should be acquired");
10689
10690 assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
10691 assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
10692 assert!(
10693 opts.namespace_lock_fence.is_some(),
10694 "set layer must receive the outer write-lock loss fence"
10695 );
10696
10697 let alpha_lock = store
10698 .handle_new_ns_lock("bucket", "alpha")
10699 .await
10700 .expect("alpha namespace lock should be created");
10701 let err = alpha_lock
10702 .get_read_lock(Duration::from_millis(20))
10703 .await
10704 .expect_err("batch delete write guard should block alpha readers");
10705 assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
10706
10707 drop(guards);
10708 alpha_lock
10709 .get_read_lock(Duration::from_secs(1))
10710 .await
10711 .expect("alpha read lock should be available after dropping batch guards");
10712 }
10713
10714 #[tokio::test]
10715 async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
10716 let store = new_read_lock_test_store().await;
10717 let mut opts = ObjectOptions::default();
10718
10719 let guard = store
10720 .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10721 .await
10722 .expect("read lock should be acquired");
10723
10724 assert!(guard.is_some(), "read lock should be held by the outer store layer");
10725 assert!(opts.no_lock, "set layer should not reacquire the object lock");
10726 assert!(
10727 opts.metadata_cache_safe,
10728 "metadata cache is safe only because the outer store layer acquired the read lock"
10729 );
10730 }
10731
10732 #[tokio::test]
10733 async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
10734 let store = new_read_lock_test_store().await;
10735 let mut opts = ObjectOptions {
10736 no_lock: true,
10737 ..Default::default()
10738 };
10739
10740 let guard = store
10741 .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10742 .await
10743 .expect("prelocked read should not acquire another lock");
10744
10745 assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
10746 assert!(
10747 !opts.metadata_cache_safe,
10748 "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
10749 );
10750 }
10751
10752 #[tokio::test]
10762 #[serial_test::serial]
10763 async fn restore_accept_guard_serializes_concurrent_accepts() {
10764 let store = Arc::new(new_read_lock_test_store().await);
10768 let _first = store
10769 .acquire_restore_accept_guard("bucket", "object")
10770 .await
10771 .expect("first accept guard should be acquired");
10772
10773 let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
10774 match store.acquire_restore_accept_guard("bucket", "object").await {
10775 Ok(_) => panic!("second accept guard must wait behind the first"),
10776 Err(err) => err,
10777 }
10778 })
10779 .await;
10780
10781 assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
10782 }
10783
10784 #[tokio::test]
10785 #[serial_test::serial]
10786 async fn restore_worker_guard_proves_generation_liveness_until_drop() {
10787 let store = Arc::new(new_read_lock_test_store().await);
10788 let operation_id = Uuid::from_u128(1);
10789
10790 let first = store
10791 .acquire_restore_worker_guard(operation_id)
10792 .await
10793 .expect("first worker must own its newly generated lock");
10794 assert!(
10795 store
10796 .try_acquire_restore_worker_guard(operation_id)
10797 .await
10798 .expect("a contended liveness probe should remain typed")
10799 .is_none(),
10800 "the same generation must remain live while its worker guard is held"
10801 );
10802
10803 drop(first);
10804 assert!(
10805 store
10806 .try_acquire_restore_worker_guard(operation_id)
10807 .await
10808 .expect("post-drop liveness probe should not fail")
10809 .is_some(),
10810 "dropping the worker guard must make the orphan generation recoverable"
10811 );
10812 }
10813
10814 #[tokio::test]
10815 #[serial_test::serial]
10816 async fn reader_lock_is_held_when_optimization_is_disabled() {
10817 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
10818 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10819 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10820 let key = rustfs_lock::ObjectKey::new("bucket", "object");
10821 let read_guard = lock
10822 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10823 .await
10824 .expect("read lock should be acquired");
10825 let read_guard = ObjectLockDiagGuard::new(
10826 read_guard,
10827 true,
10828 "test_get_object",
10829 Some("bucket".to_string()),
10830 Some("object".to_string()),
10831 Some("reader".to_string()),
10832 ObjectLockDiagMode::Read,
10833 );
10834 let reader = GetObjectReader {
10835 stream: Box::new(Cursor::new(Vec::<u8>::new())),
10836 object_info: ObjectInfo::default(),
10837 buffered_body: None,
10838 body_source: Default::default(),
10839 };
10840
10841 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10842
10843 lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
10844 .await
10845 .expect_err("reader should hold the read lock");
10846 drop(reader);
10847 lock.get_write_lock(key, "writer", Duration::from_secs(1))
10848 .await
10849 .expect("dropping the reader should release the read lock");
10850 })
10851 .await;
10852 }
10853
10854 #[tokio::test]
10855 #[serial_test::serial]
10856 async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
10857 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
10858 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10859 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10860 let key = rustfs_lock::ObjectKey::new("bucket", "object");
10861 let read_guard = lock
10862 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10863 .await
10864 .expect("read lock should be acquired");
10865 let read_guard = ObjectLockDiagGuard::new(
10866 read_guard,
10867 true,
10868 "test_get_object",
10869 Some("bucket".to_string()),
10870 Some("object".to_string()),
10871 Some("reader".to_string()),
10872 ObjectLockDiagMode::Read,
10873 );
10874 let reader = GetObjectReader {
10875 stream: Box::new(Cursor::new(vec![1, 2, 3])),
10876 object_info: ObjectInfo::default(),
10877 buffered_body: None,
10878 body_source: Default::default(),
10879 };
10880
10881 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10882
10883 lock.get_write_lock(key, "writer", Duration::from_secs(1))
10884 .await
10885 .expect("lock optimization should release the read lock before returning the stream");
10886 drop(reader);
10887 })
10888 .await;
10889 }
10890
10891 #[tokio::test]
10892 #[serial_test::serial]
10893 async fn reader_lock_is_not_held_for_buffered_body_when_optimization_is_enabled() {
10894 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
10895 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10896 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10897 let key = rustfs_lock::ObjectKey::new("bucket", "object");
10898 let read_guard = lock
10899 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10900 .await
10901 .expect("read lock should be acquired");
10902 let read_guard = ObjectLockDiagGuard::new(
10903 read_guard,
10904 true,
10905 "test_get_object",
10906 Some("bucket".to_string()),
10907 Some("object".to_string()),
10908 Some("reader".to_string()),
10909 ObjectLockDiagMode::Read,
10910 );
10911 let reader = GetObjectReader {
10912 stream: Box::new(Cursor::new(vec![1, 2, 3])),
10913 object_info: ObjectInfo::default(),
10914 buffered_body: Some(Bytes::from_static(b"123")),
10915 body_source: Default::default(),
10916 };
10917
10918 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10919
10920 lock.get_write_lock(key, "writer", Duration::from_secs(1))
10921 .await
10922 .expect("buffered reader should release the read lock immediately");
10923 drop(reader);
10924 })
10925 .await;
10926 }
10927
10928 #[tokio::test]
10929 #[serial_test::serial]
10930 async fn reader_lock_is_released_after_stream_eof() {
10931 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
10932 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10933 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10934 let key = rustfs_lock::ObjectKey::new("bucket", "object");
10935 let read_guard = lock
10936 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10937 .await
10938 .expect("read lock should be acquired");
10939 let read_guard = ObjectLockDiagGuard::new(
10940 read_guard,
10941 true,
10942 "test_get_object",
10943 Some("bucket".to_string()),
10944 Some("object".to_string()),
10945 Some("reader".to_string()),
10946 ObjectLockDiagMode::Read,
10947 );
10948 let reader = GetObjectReader {
10949 stream: Box::new(Cursor::new(vec![1, 2, 3])),
10950 object_info: ObjectInfo::default(),
10951 buffered_body: None,
10952 body_source: Default::default(),
10953 };
10954
10955 let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10956 let mut output = Vec::new();
10957 reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
10958 assert_eq!(output, vec![1, 2, 3]);
10959
10960 lock.get_write_lock(key, "writer", Duration::from_secs(1))
10961 .await
10962 .expect("EOF should release the read lock before the reader is dropped");
10963 drop(reader);
10964 })
10965 .await;
10966 }
10967}