1use super::*;
16use crate::bucket::lifecycle::{
17 bucket_lifecycle_ops::{enqueue_committed_free_versions, eval_action_from_lifecycle},
18 get_expiry_configs,
19 tier_delete_journal::{
20 ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
21 TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
22 inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
23 record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
24 tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
25 },
26 tier_sweeper::{
27 Jentry, TierDeleteSourceIdentity, attach_tier_delete_source, transitioned_delete_journal_entry_for_source,
28 transitioned_force_delete_journal_entry,
29 },
30};
31use crate::bucket::metadata_sys::{
32 acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
33 get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
34};
35use crate::bucket::object_lock::objectlock_sys::{
36 check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
37};
38use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
39use crate::bucket::versioning::VersioningApi;
40use crate::core::pools::{DecommissionCapacityAdmission, DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
41use crate::diagnostics::object_lock::ObjectLockAttempt;
42use crate::disk::OldCurrentSize;
43use crate::object_api::{
44 NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
45 TierFreeVersionReceiptSink,
46};
47use crate::services::notification_sys::{
48 TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
49 tier_delete_journal_topology_generation,
50};
51use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
52use crate::set_disk::{
53 SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
54 is_lock_optimization_enabled, is_object_lock_diag_enabled,
55};
56use crate::storage_api_contracts::{
57 list::ListOperations as _,
58 namespace::NamespaceLocking as _,
59 object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
60};
61use futures::StreamExt as _;
62use parking_lot::Mutex as ParkingMutex;
63use rustfs_filemeta::ObjectPartInfo;
64use rustfs_io_metrics::{
65 record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
66 record_object_lock_diag_slow_hold,
67};
68use std::{
69 fmt,
70 pin::Pin,
71 sync::atomic::{AtomicBool, Ordering},
72 task::{Context, Poll},
73 time::{Duration, Instant},
74};
75use tokio::io::{AsyncRead, ReadBuf};
76
77#[cfg(not(test))]
78const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
79#[cfg(test)]
80const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
81const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
82const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
83const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
84
85fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
86 if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
87 return None;
88 }
89
90 let sink = TierFreeVersionReceiptSink::new();
91 opts.tier_free_version_receipt_sink = Some(sink.clone());
92 Some(sink)
93}
94
95async fn enqueue_recorded_tier_free_versions(store: &ECStore, sink: Option<TierFreeVersionReceiptSink>) -> usize {
96 let Some(sink) = sink else {
97 return 0;
98 };
99 let Ok(receipts) = sink.drain() else {
100 return 0;
101 };
102 enqueue_committed_free_versions(store, receipts).await
103}
104
105fn build_tier_delete_journal_entry(
106 bucket: &str,
107 object: &str,
108 opts: &ObjectOptions,
109 source: &ObjectInfo,
110) -> Result<Option<Jentry>> {
111 let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
112 let source_object = decode_dir_object(object);
113 let entry = if opts.delete_prefix {
114 transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
115 attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
116 je
117 })
118 } else {
119 transitioned_delete_journal_entry_for_source(
120 version_id,
121 opts.versioned,
122 opts.version_suspended,
123 bucket,
124 source_object.as_str(),
125 source,
126 )
127 };
128 let Some(mut je) = entry else {
129 let lifecycle_exact_object_delete_all = opts.delete_prefix_object && opts.lifecycle_delete_all.is_some();
130 if opts.delete_prefix
131 && !lifecycle_exact_object_delete_all
132 && source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE
133 {
134 return Err(Error::other(
135 "recursive prefix delete cannot preserve a transitioned source with legacy Unknown remote-version state",
136 ));
137 }
138 return Ok(None);
139 };
140 record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
141 if opts.delete_prefix && !je.can_replace_tier_free_version() {
142 return Err(Error::other(
143 "recursive prefix delete requires a stable transitioned source and destination identity",
144 ));
145 }
146 Ok(Some(je))
147}
148
149async fn prepare_prefix_tier_delete_journal_entries(
150 api: &Arc<ECStore>,
151 bucket: &str,
152 prefix: &str,
153 opts: &ObjectOptions,
154) -> Result<PreparedPrefixTierDelete> {
155 Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
156}
157
158type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
159
160fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
161 truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
162}
163
164fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
165 let mut cancelled = false;
166 for result in results {
167 match result {
168 Ok(()) => {}
169 Err(Error::OperationCanceled) => cancelled = true,
170 Err(err) => return Err(err),
171 }
172 }
173 if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
174}
175
176async fn acquire_prefix_tier_delete_reference_leases(
177 api: &Arc<ECStore>,
178 tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
179) -> Result<Vec<TierOperationLease>> {
180 let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
181 tier_references.sort_unstable();
182 let mut leases = Vec::with_capacity(tier_references.len());
183 for (tier_name, backend_identity) in tier_references {
184 let lease = match backend_identity {
185 Some(backend_identity) => {
186 TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
187 .await
188 }
189 None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
190 }
191 .map_err(Error::other)?;
192 leases.push(lease);
193 }
194 Ok(leases)
195}
196
197async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
198 let tier_references = entries
199 .iter()
200 .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
201 .collect::<std::collections::HashSet<_>>();
202 acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
203}
204
205async fn prepare_prefix_tier_delete_journal_entries_inner(
206 api: &Arc<ECStore>,
207 bucket: &str,
208 prefix: &str,
209 opts: &ObjectOptions,
210) -> Result<PreparedPrefixTierDelete> {
211 let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
212 (false, false, None)
213 } else {
214 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
215 let bucket_fence = opts
216 .bucket_lifecycle_lock_fence
217 .as_ref()
218 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
219 match Box::pin(inspect_tier_delete_chunk_parent(
220 Arc::clone(api),
221 bucket,
222 bucket_incarnation,
223 prefix,
224 bucket_fence,
225 ))
226 .await?
227 {
228 TierDeleteChunkParentInspection::NoParent => (false, false, None),
229 TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
230 TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
231 TierDeleteChunkParentInspection::Resume(dispatch) => {
232 let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
233 return Ok(PreparedPrefixTierDelete {
234 dispatch: Some(*dispatch),
235 chunk_parent_active: true,
236 chunk_parent_fleet_proof: None,
237 _leases: leases,
238 });
239 }
240 TierDeleteChunkParentInspection::RetryRequired => {
241 return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
242 }
243 }
244 };
245 let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
246 let mut entries_by_name = std::collections::BTreeMap::new();
247 let batch_limit = tier_delete_dispatch_batch_limit();
248 let logical_prefix = decode_dir_object(prefix);
249 let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
250 let physical_sets = api
251 .pools
252 .iter()
253 .flat_map(|pool| pool.disk_set.iter().cloned())
254 .collect::<Vec<_>>();
255 let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
256 let cancellation = tokio_util::sync::CancellationToken::new();
257 let limit_cancellation = Arc::new(AtomicBool::new(false));
258 let walk_cancel = cancellation.clone();
259 let bucket_owned = bucket.to_string();
260 let prefix_owned = prefix.to_string();
261 let walk = async move {
262 let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
263 let tx = tx.clone();
264 let cancellation = walk_cancel.clone();
265 let bucket = bucket_owned.clone();
266 let prefix = prefix_owned.clone();
267 async move {
268 let result = set
269 .walk(
270 cancellation.clone(),
271 &bucket,
272 &prefix,
273 tx,
274 WalkOptions {
275 include_free_versions: true,
276 walkdir_timeout: Some(Duration::ZERO),
277 ..Default::default()
278 },
279 )
280 .await;
281 if result.is_err() {
282 cancellation.cancel();
283 }
284 result
285 }
286 }))
287 .buffer_unordered(4)
288 .collect::<Vec<_>>()
289 .await;
290 drop(tx);
291 combine_tier_delete_walk_results(results)
292 };
293 let collect_limit_cancellation = limit_cancellation.clone();
294 let collect = async {
295 let mut truncated = false;
296 while let Some(result) = rx.recv().await {
297 if let Some(err) = result.err {
298 if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
304 {
305 continue;
306 }
307 cancellation.cancel();
308 return Err(err);
309 }
310 if truncated {
311 continue;
312 }
313 let Some(source) = result.item else {
314 continue;
315 };
316 let object = decode_dir_object(&source.name);
317 if exact_object.as_ref().is_some_and(|expected| expected != &object) {
318 continue;
319 }
320 if exact_object.is_none() && !object.starts_with(&logical_prefix) {
321 continue;
322 }
323 if source.transitioned_object.free_version {
324 cancellation.cancel();
325 return Err(Error::other(
326 "recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
327 ));
328 }
329 let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
330 let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
331 Some((source.transitioned_object.tier.clone(), backend_identity))
332 } else {
333 None
334 };
335 if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
336 let name = tier_delete_journal_object_name(&entry);
337 let at_limit = entries_by_name.len() == batch_limit;
338 match entries_by_name.entry(name) {
339 std::collections::btree_map::Entry::Occupied(_) => {}
340 std::collections::btree_map::Entry::Vacant(_) if at_limit => {
341 truncated = true;
342 collect_limit_cancellation.store(true, Ordering::Release);
343 cancellation.cancel();
344 }
345 std::collections::btree_map::Entry::Vacant(slot) => {
346 if let Some(tier_reference) = tier_reference {
347 tier_references.insert(tier_reference);
348 }
349 slot.insert(entry);
350 }
351 }
352 } else if let Some(tier_reference) = tier_reference {
353 tier_references.insert(tier_reference);
354 }
355 }
356 Ok(truncated)
357 };
358 let (walk_result, collect_result) = tokio::join!(walk, collect);
359 let truncated = collect_result?;
360 if let Err(err) = walk_result
364 && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
365 {
366 return Err(err);
367 }
368 let entries = entries_by_name.into_values().collect::<Vec<_>>();
369 let mut leased_tier_references = tier_references;
370 let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
371 if entries.is_empty() {
372 let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
373 let fleet_proof = acquire_tier_delete_journal_fleet_proof()
374 .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
375 if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
376 return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
377 }
378 Some(fleet_proof)
379 } else {
380 None
381 };
382 return Ok(PreparedPrefixTierDelete {
383 dispatch: None,
384 chunk_parent_active,
385 chunk_parent_fleet_proof,
386 _leases: leases,
387 });
388 }
389
390 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
391 let bucket_fence = opts
392 .bucket_lifecycle_lock_fence
393 .as_ref()
394 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
395 let fleet_proof = acquire_tier_delete_journal_fleet_proof()
396 .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
397 if chunk_parent_topology_generation
398 .as_deref()
399 .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
400 {
401 return Err(Error::other("tier delete chunk parent topology changed during source scan"));
402 }
403 let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
404 Box::pin(prepare_tier_delete_chunk_dispatch(
405 Arc::clone(api),
406 bucket,
407 bucket_incarnation,
408 prefix,
409 entries,
410 truncated && !chunk_parent_active,
411 fleet_proof,
412 bucket_fence,
413 ))
414 .await?
415 } else if legacy_manifest_active && truncated {
416 resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
417 .await?
418 } else {
419 prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
420 .await?
421 };
422 let additional_tier_references = dispatch
427 .entries()?
428 .iter()
429 .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
430 .filter(|reference| leased_tier_references.insert(reference.clone()))
431 .collect::<std::collections::HashSet<_>>();
432 leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
433 if legacy_manifest_active && truncated {
434 dispatch.require_exact_predecessor_replay();
435 }
436 Ok(PreparedPrefixTierDelete {
437 dispatch: Some(dispatch),
438 chunk_parent_active: chunk_parent_active || truncated,
439 chunk_parent_fleet_proof: None,
440 _leases: leases,
441 })
442}
443
444struct PreparedPrefixTierDelete {
445 dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
446 chunk_parent_active: bool,
447 chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
448 _leases: Vec<TierOperationLease>,
449}
450
451#[cfg(all(test, feature = "test-util"))]
452struct PrefixDeleteAfterJournalPrepareBarrierState {
453 bucket: String,
454 prefix: String,
455 arrived: tokio::sync::Notify,
456 release: tokio::sync::Notify,
457}
458
459#[cfg(all(test, feature = "test-util"))]
460pub(crate) struct PrefixDeleteAfterJournalPrepareBarrier {
461 state: Arc<PrefixDeleteAfterJournalPrepareBarrierState>,
462}
463
464#[cfg(all(test, feature = "test-util"))]
465static PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER: std::sync::OnceLock<
466 std::sync::Mutex<Option<Arc<PrefixDeleteAfterJournalPrepareBarrierState>>>,
467> = std::sync::OnceLock::new();
468
469#[cfg(all(test, feature = "test-util"))]
470impl PrefixDeleteAfterJournalPrepareBarrier {
471 pub(crate) fn install(bucket: &str, prefix: &str) -> Self {
472 let state = Arc::new(PrefixDeleteAfterJournalPrepareBarrierState {
473 bucket: bucket.to_string(),
474 prefix: prefix.to_string(),
475 arrived: tokio::sync::Notify::new(),
476 release: tokio::sync::Notify::new(),
477 });
478 let mut slot = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
479 .get_or_init(|| std::sync::Mutex::new(None))
480 .lock()
481 .expect("prefix journal barrier mutex should not poison");
482 assert!(slot.is_none(), "prefix journal barrier must not already be installed");
483 *slot = Some(Arc::clone(&state));
484 Self { state }
485 }
486
487 pub(crate) async fn wait_until_paused(&self) {
488 self.state.arrived.notified().await;
489 }
490
491 pub(crate) fn release(&self) {
492 self.state.release.notify_one();
493 }
494}
495
496#[cfg(all(test, feature = "test-util"))]
497impl Drop for PrefixDeleteAfterJournalPrepareBarrier {
498 fn drop(&mut self) {
499 self.state.release.notify_one();
500 if let Some(slot) = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER.get() {
501 let mut slot = slot.lock().expect("prefix journal barrier mutex should not poison");
502 if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
503 *slot = None;
504 }
505 }
506 }
507}
508
509#[cfg(all(test, feature = "test-util"))]
510async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) {
511 let state = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
512 .get_or_init(|| std::sync::Mutex::new(None))
513 .lock()
514 .expect("prefix journal barrier mutex should not poison")
515 .as_ref()
516 .filter(|state| state.bucket == bucket && state.prefix == prefix)
517 .cloned();
518 if let Some(state) = state {
519 state.arrived.notify_one();
520 state.release.notified().await;
521 }
522}
523
524async fn delete_prefix_with_tier_delete_journal(
525 store: &ECStore,
526 bucket: &str,
527 object: &str,
528 opts: &ObjectOptions,
529 tier_journal_api: Option<&Arc<ECStore>>,
530) -> Result<()> {
531 let Some(api) = tier_journal_api else {
532 return store.delete_prefix(bucket, object, opts).await;
533 };
534 let PreparedPrefixTierDelete {
535 dispatch,
536 chunk_parent_active,
537 chunk_parent_fleet_proof,
538 _leases,
539 } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
540 let Some(dispatch) = dispatch else {
541 let _tier_leases = _leases;
545 let mut operation_opts = opts.clone();
546 operation_opts.tier_delete_journal_api = None;
551 operation_opts.tier_delete_dispatch_authorization = None;
552 let parent_fleet_proof = if chunk_parent_active {
553 Some(
554 chunk_parent_fleet_proof
555 .as_ref()
556 .filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
557 .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
558 )
559 } else {
560 None
561 };
562 store.delete_prefix(bucket, object, &operation_opts).await?;
563 if let Some(parent_fleet_proof) = parent_fleet_proof {
564 #[cfg(all(test, feature = "test-util"))]
565 crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
566 crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
567 )
568 .await;
569 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
570 let bucket_fence = opts
571 .bucket_lifecycle_lock_fence
572 .as_ref()
573 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
574 if !Box::pin(complete_tier_delete_chunk_parent(
575 Arc::clone(api),
576 bucket,
577 bucket_incarnation,
578 object,
579 bucket_fence,
580 parent_fleet_proof,
581 ))
582 .await?
583 {
584 return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
585 }
586 }
587 return Ok(());
588 };
589 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
590 let bucket_fence = opts
591 .bucket_lifecycle_lock_fence
592 .as_ref()
593 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
594 let active = dispatch.consume(bucket, bucket_incarnation, object)?;
595 let mut operation_opts = opts.clone();
596 operation_opts.tier_delete_dispatch_authorization = Some(active.authorization());
597
598 #[cfg(all(test, feature = "test-util"))]
599 pause_prefix_delete_after_journal_prepare(bucket, object).await;
600
601 let _tier_leases = _leases;
604 if active.predecessor_replay_required() || active.is_chunked() {
605 replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
606 #[cfg(all(test, feature = "test-util"))]
607 if active.is_chunked() {
608 crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
609 crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
610 )
611 .await;
612 }
613 complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
614 if active.is_chunked() {
615 return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
616 }
617 return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
618 }
619 let result = store.delete_prefix(bucket, object, &operation_opts).await;
620 match result {
621 Ok(()) => {
622 if let Err(err) = complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await {
623 warn!(
624 event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
625 component = LOG_COMPONENT_ECSTORE,
626 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
627 bucket,
628 prefix = object,
629 error = ?err,
630 "Prefix deletion committed with durable tier cleanup pending"
631 );
632 }
633 Ok(())
634 }
635 Err(err) => Err(err),
638 }
639}
640
641async fn replay_authorized_tier_delete_sources(
642 store: &ECStore,
643 bucket: &str,
644 prefix: &str,
645 active: &ActiveTierDeleteDispatch,
646 opts: &ObjectOptions,
647) -> Result<()> {
648 let _publication_scope_guard = opts
649 .scanner_publication_commit_scope
650 .clone()
651 .map(ScannerPublicationCommitScopeGuard::new);
652 let publication_scope = opts.scanner_publication_commit_scope.as_ref();
653 let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
654 let bucket_fence = opts
655 .bucket_lifecycle_lock_fence
656 .as_ref()
657 .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
658 let authorization = active.authorization();
659 authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
660
661 let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
662 for entry in active.entries() {
663 let source = entry
664 .source
665 .as_ref()
666 .filter(|source| source.has_stable_identity())
667 .ok_or_else(|| Error::other("authorized tier delete predecessor has no stable source"))?;
668 if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
669 return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
670 }
671 source_objects.insert(source.object.as_str());
672 }
673
674 if let Some(scope) = publication_scope {
675 if scope.state() == ScannerPublicationCommitState::Admitted {
676 scope
677 .try_begin()
678 .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
679 }
680 if !scope.can_commit() {
681 let _ = scope.mark_indeterminate();
682 return Err(StorageError::OperationCanceled);
683 }
684 }
685
686 let stopped = Arc::new(AtomicBool::new(false));
687 let make_replay = |object: String| {
691 let stopped = stopped.clone();
692 let authorization = authorization.clone();
693 async move {
694 if stopped.load(Ordering::Acquire) {
695 return Ok::<_, Error>(0usize);
696 }
697 let result = async {
698 if bucket_fence.is_lock_lost() {
699 return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
700 }
701 let encoded_object = encode_dir_object(&object);
702 let guards = if opts.delete_prefix_object {
703 store
704 .acquire_remaining_physical_object_write_locks(
705 "tier_delete_dispatch_predecessor_replay",
706 bucket,
707 &encoded_object,
708 )
709 .await?
710 } else {
711 store
712 .acquire_all_physical_object_write_locks(
713 "tier_delete_dispatch_predecessor_replay",
714 bucket,
715 &encoded_object,
716 )
717 .await?
718 };
719 authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
720 if publication_scope.is_some_and(|scope| !scope.can_commit()) {
721 return Err(StorageError::OperationCanceled);
722 }
723 let mut replay_opts = opts.clone();
724 replay_opts.no_lock = true;
725 replay_opts.delete_prefix = false;
726 replay_opts.delete_prefix_object = false;
727 for guard in &guards {
728 guard.add_namespace_lock_fence(&mut replay_opts);
729 }
730 let mut deleted = 0usize;
731 for pool in &store.pools {
732 for set in &pool.disk_set {
733 authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
734 deleted = deleted
735 .checked_add(
736 set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
737 .await?,
738 )
739 .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
740 }
741 }
742 if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
743 return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
744 }
745 Ok(deleted)
746 }
747 .await;
748 if result.is_err() {
749 stopped.store(true, Ordering::Release);
750 }
751 result
752 }
753 };
754 let mut objects = source_objects.into_iter();
755 let mut replays = futures::stream::FuturesUnordered::new();
756 for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
757 let Some(object) = objects.next().map(ToOwned::to_owned) else {
758 break;
759 };
760 replays.push(make_replay(object));
761 }
762 let mut deleted = 0usize;
763 let mut first_error = None;
764 while let Some(result) = replays.next().await {
765 match result {
766 Ok(count) => {
767 deleted = deleted
768 .checked_add(count)
769 .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
770 }
771 Err(err) if first_error.is_none() => first_error = Some(err),
772 Err(_) => {}
773 }
774 if first_error.is_none()
775 && !stopped.load(Ordering::Acquire)
776 && let Some(object) = objects.next().map(ToOwned::to_owned)
777 {
778 replays.push(make_replay(object));
779 }
780 }
781 if deleted > 0 {
782 super::list_objects::observe_list_objects_mutation(store, bucket).await;
783 }
784 if let Some(err) = first_error {
785 if publication_scope.is_some_and(|scope| !scope.can_commit())
786 && let Some(scope) = publication_scope
787 {
788 let _ = scope.mark_indeterminate();
789 }
790 return Err(err);
791 }
792 if let Some(scope) = publication_scope {
793 let _ = scope.mark_committed();
794 }
795 Ok(())
796}
797
798fn tier_delete_source_matches_replay_scope(
799 source: &TierDeleteSourceIdentity,
800 bucket: &str,
801 persisted_prefix: &str,
802 exact_object: bool,
803) -> bool {
804 tier_delete_source_matches_dispatch_scope(source, bucket, persisted_prefix)
805 && (!exact_object || source.object == decode_dir_object(persisted_prefix))
806}
807
808async fn delete_recursive_prefix_with_tier_delete_journal(
809 store: &ECStore,
810 bucket: &str,
811 object: &str,
812 opts: &ObjectOptions,
813 tier_journal_api: Option<&Arc<ECStore>>,
814) -> Result<()> {
815 if !is_meta_bucketname(bucket) {
818 let state = opts
819 .object_lock_config_snapshot
820 .as_deref()
821 .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))?
822 .state();
823 ensure_recursive_force_delete_allowed_for_state(bucket, state)?;
824 let bypass_governance = opts
825 .object_lock_delete
826 .as_ref()
827 .is_some_and(|delete_opts| delete_opts.bypass_governance);
828 for pool in &store.pools {
829 for set in &pool.disk_set {
830 let mut marker = None;
831 let mut version_marker = None;
832 loop {
833 let page = set
834 .clone()
835 .inner_list_object_versions_for_recursive_delete(
836 bucket,
837 object,
838 marker.clone(),
839 version_marker.clone(),
840 RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
841 )
842 .await?;
843 for object_info in &page.objects {
844 if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() {
845 return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone()));
846 }
847 }
848 if !page.is_truncated {
849 break;
850 }
851 let next_marker = page
852 .next_marker
853 .ok_or_else(|| Error::other("recursive delete version scan did not return a continuation marker"))?;
854 if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker {
855 return Err(Error::other("recursive delete version scan did not advance"));
856 }
857 marker = Some(next_marker);
858 version_marker = page.next_version_idmarker;
859 }
860 }
861 }
862 }
863 if object.ends_with('/') && !is_meta_bucketname(bucket) {
868 let parent = object.strip_suffix('/').unwrap_or(object);
869 for pool in &store.pools {
870 for set in &pool.disk_set {
871 let page = set
872 .clone()
873 .inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
874 .await?;
875 if page.objects.iter().any(|info| info.name == parent) {
876 return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
877 }
878 }
879 }
880 }
881 delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
882}
883
884async fn delete_directory_keys_with_tier_delete_journal(
885 store: &ECStore,
886 bucket: &str,
887 prefix: &str,
888 opts: &ObjectOptions,
889 tier_journal_api: Option<&Arc<ECStore>>,
890) -> Result<()> {
891 for pool in &store.pools {
892 for set in &pool.disk_set {
893 let mut previous_keys = std::collections::BTreeSet::new();
894 loop {
895 let page = set
898 .clone()
899 .inner_list_object_versions_for_recursive_delete(
900 bucket,
901 prefix,
902 None,
903 None,
904 RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
905 )
906 .await?;
907 let keys = page
908 .objects
909 .into_iter()
910 .map(|info| info.name)
911 .filter(|key| key.starts_with(prefix))
912 .collect::<std::collections::BTreeSet<_>>();
913 if keys.is_empty() {
914 break;
915 }
916 if keys == previous_keys {
917 return Err(Error::other("directory deletion did not advance"));
918 }
919 for key in &keys {
920 let encoded_key = encode_dir_object(key);
921 let mut exact_opts = opts.clone();
922 exact_opts.delete_prefix_object = true;
923 let _guard = store
924 .acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
925 .await?;
926 delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
927 }
928 previous_keys = keys;
929 }
930 }
931 }
932 Ok(())
933}
934
935pub struct PreparedGetObjectReader {
942 pool: Arc<Sets>,
943 bucket: String,
944 object: String,
945 range: Option<HTTPRangeSpec>,
946 headers: HeaderMap,
947 opts: ObjectOptions,
948 metadata: crate::set_disk::PreparedGetObjectMetadata,
949 read_lock_guard: Option<ObjectLockDiagGuard>,
950}
951
952impl PreparedGetObjectReader {
953 pub fn object_info(&self) -> &ObjectInfo {
955 self.metadata.object_info()
956 }
957
958 pub fn into_object_info(mut self) -> ObjectInfo {
960 self.metadata.take_object_info()
961 }
962
963 #[must_use]
965 pub fn with_headers(mut self, headers: HeaderMap) -> Self {
966 self.headers = headers;
967 self
968 }
969
970 pub async fn into_reader(self) -> Result<GetObjectReader> {
975 let mut reader =
976 crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
977 &self.bucket,
978 &self.object,
979 self.range,
980 self.headers,
981 &self.opts,
982 self.metadata,
983 ))
984 .await?;
985 reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
986 Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
987 }
988}
989
990struct LockGuardedReader {
991 inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
992 guard: Option<ObjectLockDiagGuard>,
993}
994
995impl AsyncRead for LockGuardedReader {
996 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
997 let had_capacity = buf.remaining() > 0;
998 let filled_before = buf.filled().len();
999 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1000 if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
1001 self.guard.take();
1002 }
1003 poll
1004 }
1005}
1006
1007#[derive(Clone, Copy, Debug)]
1008enum ObjectLockDiagMode {
1009 Read,
1010 Write,
1011}
1012
1013impl ObjectLockDiagMode {
1014 fn as_str(self) -> &'static str {
1015 match self {
1016 Self::Read => "read",
1017 Self::Write => "write",
1018 }
1019 }
1020}
1021
1022impl fmt::Display for ObjectLockDiagMode {
1023 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1024 f.write_str(self.as_str())
1025 }
1026}
1027
1028pub(crate) struct ObjectLockDiagGuard {
1029 guard: rustfs_lock::NamespaceLockGuard,
1030 #[cfg(test)]
1031 test_namespace_lock_fence: Option<NamespaceLockFence>,
1032 enabled: bool,
1033 op: &'static str,
1034 bucket: Option<String>,
1035 object: Option<String>,
1036 owner: Option<String>,
1037 mode: ObjectLockDiagMode,
1038 acquired_at: Instant,
1039 _attempt: ObjectLockAttempt,
1040}
1041
1042impl ObjectLockDiagGuard {
1043 fn new(
1044 guard: rustfs_lock::NamespaceLockGuard,
1045 enabled: bool,
1046 op: &'static str,
1047 bucket: Option<String>,
1048 object: Option<String>,
1049 owner: Option<String>,
1050 mode: ObjectLockDiagMode,
1051 ) -> Self {
1052 Self {
1053 guard,
1054 #[cfg(test)]
1055 test_namespace_lock_fence: None,
1056 enabled,
1057 op,
1058 bucket,
1059 object,
1060 owner,
1061 mode,
1062 acquired_at: Instant::now(),
1063 _attempt: ObjectLockAttempt::default(),
1064 }
1065 }
1066
1067 fn with_attempt(mut self, attempt: ObjectLockAttempt) -> Self {
1068 self._attempt = attempt;
1069 self
1070 }
1071
1072 pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
1073 match &self.guard {
1074 rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
1075 rustfs_lock::NamespaceLockGuard::Fast(_) => None,
1076 }
1077 }
1078
1079 pub(crate) fn is_lock_lost(&self) -> bool {
1080 self.guard.is_lock_lost() || {
1081 #[cfg(test)]
1082 {
1083 self.test_namespace_lock_fence
1084 .as_ref()
1085 .is_some_and(NamespaceLockFence::is_lock_lost)
1086 }
1087 #[cfg(not(test))]
1088 {
1089 false
1090 }
1091 }
1092 }
1093
1094 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1095 opts.ensure_namespace_lock_fence();
1096 if let Some(signal) = self.lock_lost_signal() {
1097 opts.add_namespace_lock_lost_signal(signal);
1098 }
1099 #[cfg(test)]
1100 if let Some(fence) = self.test_namespace_lock_fence.as_ref() {
1101 opts.add_namespace_lock_fence(fence);
1102 }
1103 }
1104}
1105
1106#[cfg(test)]
1107#[derive(Clone, Copy, PartialEq, Eq)]
1108pub(crate) enum DecommissionMutationFenceTestPhase {
1109 Migration,
1110 SourceCleanup,
1111}
1112
1113#[cfg(test)]
1114struct DecommissionMutationFenceLossState {
1115 bucket: String,
1116 object: String,
1117 phase: DecommissionMutationFenceTestPhase,
1118 fence: NamespaceLockFence,
1119 loss_handle: Arc<std::sync::atomic::AtomicBool>,
1120}
1121
1122#[cfg(test)]
1123pub(crate) struct DecommissionMutationFenceLossHook {
1124 state: Arc<DecommissionMutationFenceLossState>,
1125}
1126
1127#[cfg(test)]
1128static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock<
1129 std::sync::Mutex<Option<Arc<DecommissionMutationFenceLossState>>>,
1130> = std::sync::OnceLock::new();
1131
1132#[cfg(test)]
1133impl DecommissionMutationFenceLossHook {
1134 pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self {
1135 let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
1136 let state = Arc::new(DecommissionMutationFenceLossState {
1137 bucket: bucket.to_string(),
1138 object: object.to_string(),
1139 phase,
1140 fence,
1141 loss_handle,
1142 });
1143 let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1144 .get_or_init(|| std::sync::Mutex::new(None))
1145 .lock()
1146 .expect("decommission mutation fence loss hooks should not poison");
1147 assert!(slot.is_none(), "decommission mutation fence loss hook must be unique");
1148 *slot = Some(Arc::clone(&state));
1149 Self { state }
1150 }
1151
1152 pub(crate) fn mark_lost(&self) {
1153 self.state.loss_handle.store(true, Ordering::Release);
1154 }
1155}
1156
1157#[cfg(test)]
1158impl Drop for DecommissionMutationFenceLossHook {
1159 fn drop(&mut self) {
1160 let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1161 .get_or_init(|| std::sync::Mutex::new(None))
1162 .lock()
1163 .expect("decommission mutation fence loss hooks should not poison");
1164 if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) {
1165 *slot = None;
1166 }
1167 }
1168}
1169
1170#[cfg(test)]
1171fn decommission_mutation_fence_for_test(
1172 bucket: &str,
1173 object: &str,
1174 phase: DecommissionMutationFenceTestPhase,
1175) -> Option<NamespaceLockFence> {
1176 DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1177 .get_or_init(|| std::sync::Mutex::new(None))
1178 .lock()
1179 .expect("decommission mutation fence loss hooks should not poison")
1180 .as_ref()
1181 .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase)
1182 .map(|hook| hook.fence.clone())
1183}
1184
1185#[cfg(all(test, feature = "test-util"))]
1186struct DecommissionFreeVersionSourceRaceState {
1187 bucket: String,
1188 object: String,
1189 arrived: tokio::sync::Notify,
1190 release: tokio::sync::Notify,
1191}
1192
1193#[cfg(all(test, feature = "test-util"))]
1194pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
1195 state: Arc<DecommissionFreeVersionSourceRaceState>,
1196}
1197
1198#[cfg(all(test, feature = "test-util"))]
1199static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
1200 std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
1201> = std::sync::OnceLock::new();
1202
1203#[cfg(all(test, feature = "test-util"))]
1204impl DecommissionFreeVersionSourceRaceBarrier {
1205 pub(crate) fn install(bucket: &str, object: &str) -> Self {
1206 let state = Arc::new(DecommissionFreeVersionSourceRaceState {
1207 bucket: bucket.to_string(),
1208 object: object.to_string(),
1209 arrived: tokio::sync::Notify::new(),
1210 release: tokio::sync::Notify::new(),
1211 });
1212 let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1213 .get_or_init(|| std::sync::Mutex::new(None))
1214 .lock()
1215 .expect("decommission free-version source race barrier should not poison");
1216 assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
1217 *slot = Some(Arc::clone(&state));
1218 Self { state }
1219 }
1220
1221 pub(crate) async fn wait_until_paused(&self) {
1222 tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1223 .await
1224 .expect("decommission should pause before acquiring the free-version source lock");
1225 }
1226
1227 pub(crate) fn release(&self) {
1228 self.state.release.notify_one();
1229 }
1230}
1231
1232#[cfg(all(test, feature = "test-util"))]
1233impl Drop for DecommissionFreeVersionSourceRaceBarrier {
1234 fn drop(&mut self) {
1235 self.state.release.notify_one();
1236 let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1237 .get_or_init(|| std::sync::Mutex::new(None))
1238 .lock()
1239 .expect("decommission free-version source race barrier should not poison");
1240 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1241 *slot = None;
1242 }
1243 }
1244}
1245
1246#[cfg(all(test, feature = "test-util"))]
1247async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
1248 let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1249 .get_or_init(|| std::sync::Mutex::new(None))
1250 .lock()
1251 .expect("decommission free-version source race barrier should not poison")
1252 .as_ref()
1253 .filter(|state| state.bucket == bucket && state.object == object)
1254 .cloned();
1255 if let Some(state) = state {
1256 state.arrived.notify_one();
1257 state.release.notified().await;
1258 }
1259}
1260
1261pub(crate) struct SourceCleanupMutationFence {
1262 guard: ObjectLockDiagGuard,
1263 source_lock_covered: bool,
1264}
1265
1266impl SourceCleanupMutationFence {
1267 pub(crate) fn source_lock_covered(&self) -> bool {
1268 self.source_lock_covered
1269 }
1270
1271 pub(crate) fn is_lock_lost(&self) -> bool {
1272 self.guard.is_lock_lost()
1273 }
1274
1275 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1276 self.guard.add_namespace_lock_fence(opts);
1277 }
1278}
1279
1280pub struct RestoreAcceptGuard(ObjectLockDiagGuard);
1285
1286impl RestoreAcceptGuard {
1287 pub fn is_lock_lost(&self) -> bool {
1291 self.0.guard.is_lock_lost()
1292 }
1293
1294 pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1295 self.0.add_namespace_lock_fence(opts);
1296 }
1297}
1298
1299pub struct RestoreWorkerGuard(rustfs_lock::NamespaceLockGuard);
1303
1304impl RestoreWorkerGuard {
1305 pub fn is_lock_lost(&self) -> bool {
1306 self.0.is_lock_lost()
1307 }
1308}
1309
1310impl Drop for ObjectLockDiagGuard {
1311 fn drop(&mut self) {
1312 if !self.enabled || self.guard.is_released() {
1313 return;
1314 }
1315
1316 let hold = self.acquired_at.elapsed();
1317 record_object_lock_diag_hold_duration(self.op, self.mode.as_str(), hold);
1318 let threshold = get_object_lock_diag_slow_hold_threshold();
1319 if hold >= threshold {
1320 record_object_lock_diag_slow_hold(self.op, self.mode.as_str());
1321 warn!(
1322 target: "rustfs_ecstore::object_lock_diag",
1323 op = self.op,
1324 bucket = %self.bucket.as_deref().unwrap_or_default(),
1325 object = %self.object.as_deref().unwrap_or_default(),
1326 mode = %self.mode,
1327 owner = %self.owner.as_deref().unwrap_or_default(),
1328 hold_ms = hold.as_millis(),
1329 threshold_ms = threshold.as_millis(),
1330 "object namespace lock held longer than threshold"
1331 );
1332 }
1333 }
1334}
1335
1336#[derive(Debug, thiserror::Error)]
1338#[non_exhaustive]
1339pub enum SnapshotConsistencyError {
1340 #[error("namespace locking is disabled for SelectObjectContent")]
1341 LockingDisabled,
1342 #[error("SelectObjectContent namespace lock was lost")]
1343 LockLost,
1344 #[error("object read semantics changed while SelectObjectContent was running")]
1345 ObjectChanged,
1346}
1347
1348#[derive(Debug, thiserror::Error)]
1350#[non_exhaustive]
1351pub enum PrepareSelectObjectSnapshotError {
1352 #[error("storage failed while preparing SelectObjectContent snapshot: {0}")]
1353 Storage(#[source] StorageError),
1354 #[error("SelectObjectContent snapshot consistency failure: {0}")]
1355 Consistency(#[source] SnapshotConsistencyError),
1356 #[error("SelectObjectContent object has invalid logical size {size}")]
1357 InvalidLogicalSize { size: i64 },
1358}
1359
1360impl From<StorageError> for PrepareSelectObjectSnapshotError {
1361 fn from(error: StorageError) -> Self {
1362 Self::Storage(error)
1363 }
1364}
1365
1366impl From<SnapshotConsistencyError> for PrepareSelectObjectSnapshotError {
1367 fn from(error: SnapshotConsistencyError) -> Self {
1368 Self::Consistency(error)
1369 }
1370}
1371
1372#[derive(Debug, thiserror::Error)]
1374#[non_exhaustive]
1375pub enum SelectObjectSnapshotReadError {
1376 #[error("storage failed while opening SelectObjectContent snapshot reader: {0}")]
1377 Storage(#[source] StorageError),
1378 #[error("SelectObjectContent snapshot consistency failure: {0}")]
1379 Consistency(#[source] SnapshotConsistencyError),
1380}
1381
1382impl From<StorageError> for SelectObjectSnapshotReadError {
1383 fn from(error: StorageError) -> Self {
1384 Self::Storage(error)
1385 }
1386}
1387
1388impl From<SnapshotConsistencyError> for SelectObjectSnapshotReadError {
1389 fn from(error: SnapshotConsistencyError) -> Self {
1390 Self::Consistency(error)
1391 }
1392}
1393
1394struct SelectObjectSnapshotLease {
1395 guards: Vec<ObjectLockDiagGuard>,
1396 lost: Arc<AtomicBool>,
1397 lock_loss: tokio::sync::watch::Sender<bool>,
1398 _monitor_shutdown: tokio::sync::watch::Sender<()>,
1399}
1400
1401impl SelectObjectSnapshotLease {
1402 fn new(guards: Vec<ObjectLockDiagGuard>) -> Self {
1403 let signals = guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal);
1404 let mut waits = signals
1405 .map(|signal| async move { signal.notified().await })
1406 .collect::<futures::stream::FuturesUnordered<_>>();
1407 let lost = Arc::new(AtomicBool::new(false));
1408 let (lock_loss, _) = tokio::sync::watch::channel(false);
1409 let (monitor_shutdown, mut shutdown_rx) = tokio::sync::watch::channel(());
1410 if !waits.is_empty() {
1411 let task_lost = Arc::clone(&lost);
1412 let task_lock_loss = lock_loss.clone();
1413 tokio::spawn(async move {
1414 tokio::select! {
1415 lost = futures::StreamExt::next(&mut waits) => {
1416 if lost.is_some() {
1417 task_lost.store(true, Ordering::Release);
1418 task_lock_loss.send_replace(true);
1419 }
1420 }
1421 _ = shutdown_rx.changed() => {}
1422 }
1423 });
1424 }
1425 Self {
1426 guards,
1427 lost,
1428 lock_loss,
1429 _monitor_shutdown: monitor_shutdown,
1430 }
1431 }
1432
1433 fn check(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1434 if self.is_lost() || self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
1435 self.lost.store(true, Ordering::Release);
1436 self.lock_loss.send_replace(true);
1437 return Err(SnapshotConsistencyError::LockLost);
1438 }
1439 Ok(())
1440 }
1441
1442 fn is_lost(&self) -> bool {
1443 self.lost.load(Ordering::Acquire)
1444 }
1445
1446 fn subscribe_lock_loss(&self) -> tokio::sync::watch::Receiver<bool> {
1447 self.lock_loss.subscribe()
1448 }
1449}
1450
1451pub struct SelectObjectSnapshot {
1453 pool: Arc<Sets>,
1454 bucket: String,
1455 object: String,
1456 headers: HeaderMap,
1457 opts: ObjectOptions,
1458 object_info: ObjectInfo,
1459 logical_size: u64,
1460 read_semantics_identity: [u8; 32],
1461 first_metadata: ParkingMutex<Option<crate::set_disk::PreparedGetObjectMetadata>>,
1462 lease: Arc<SelectObjectSnapshotLease>,
1463}
1464
1465impl fmt::Debug for SelectObjectSnapshot {
1466 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1467 formatter
1468 .debug_struct("SelectObjectSnapshot")
1469 .field("bucket", &self.bucket)
1470 .field("object", &self.object)
1471 .field("logical_size", &self.logical_size)
1472 .finish_non_exhaustive()
1473 }
1474}
1475
1476impl SelectObjectSnapshot {
1477 pub fn is_for(&self, bucket: &str, object: &str) -> bool {
1478 self.bucket == bucket && self.object == encode_dir_object(object)
1479 }
1480
1481 pub fn object_info(&self) -> &ObjectInfo {
1482 &self.object_info
1483 }
1484
1485 pub fn logical_size(&self) -> u64 {
1486 self.logical_size
1487 }
1488
1489 pub fn matches_version(&self, requested: &str) -> bool {
1490 select_snapshot_version_matches(self.object_info.version_id, requested)
1491 }
1492
1493 pub fn ensure_valid(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1494 self.lease.check()
1495 }
1496
1497 #[cfg(feature = "test-util")]
1498 pub fn mark_lost_for_test(&self) {
1499 self.lease.lost.store(true, Ordering::Release);
1500 self.lease.lock_loss.send_replace(true);
1501 }
1502
1503 pub async fn open_reader(
1504 &self,
1505 range: Option<HTTPRangeSpec>,
1506 ) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
1507 self.lease.check()?;
1508 let first_metadata = self.first_metadata.lock().take();
1509 let metadata = match first_metadata {
1510 Some(metadata) => metadata,
1511 None => {
1512 self.pool
1513 .prepare_get_object_reader_metadata(&self.bucket, &self.object, &self.opts)
1514 .await?
1515 }
1516 };
1517 self.lease.check()?;
1518 if metadata.read_semantics_identity() != self.read_semantics_identity {
1519 return Err(SnapshotConsistencyError::ObjectChanged.into());
1520 }
1521
1522 let mut reader =
1523 crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
1524 &self.bucket,
1525 &self.object,
1526 range,
1527 self.headers.clone(),
1528 &self.opts,
1529 metadata,
1530 ))
1531 .await?;
1532 self.lease.check()?;
1533 reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
1534 reader.stream = Box::new(SelectObjectSnapshotReader {
1535 inner: reader.stream,
1536 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(self.lease.subscribe_lock_loss()),
1537 lease: Arc::clone(&self.lease),
1538 });
1539 Ok(reader)
1540 }
1541}
1542
1543fn select_snapshot_version_matches(actual: Option<Uuid>, requested: &str) -> bool {
1544 let requested = requested.trim();
1545 let requested = if requested.eq_ignore_ascii_case("null") {
1546 Uuid::nil()
1547 } else if let Ok(requested) = Uuid::parse_str(requested) {
1548 requested
1549 } else {
1550 return false;
1551 };
1552 actual.unwrap_or_else(Uuid::nil) == requested
1553}
1554
1555struct SelectObjectSnapshotReader {
1556 inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
1557 lock_loss_wake: SelectObjectSnapshotLockLossWake,
1558 lease: Arc<SelectObjectSnapshotLease>,
1559}
1560
1561struct SelectObjectSnapshotLockLossWake {
1562 stream: tokio_stream::wrappers::WatchStream<bool>,
1563}
1564
1565impl SelectObjectSnapshotLockLossWake {
1566 fn new(receiver: tokio::sync::watch::Receiver<bool>) -> Self {
1567 Self {
1568 stream: tokio_stream::wrappers::WatchStream::new(receiver),
1569 }
1570 }
1571
1572 fn poll_lost(&mut self, cx: &mut Context<'_>) -> bool {
1573 loop {
1574 match futures::Stream::poll_next(Pin::new(&mut self.stream), cx) {
1575 Poll::Ready(Some(true)) => return true,
1576 Poll::Ready(Some(false)) => {}
1577 Poll::Ready(None) | Poll::Pending => return false,
1578 }
1579 }
1580 }
1581}
1582
1583impl AsyncRead for SelectObjectSnapshotReader {
1584 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
1585 if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() {
1586 return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1587 }
1588 let filled_before = buf.filled().len();
1589 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1590 if self.lease.check().is_err() {
1591 buf.set_filled(filled_before);
1592 return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1593 }
1594 poll
1595 }
1596}
1597
1598fn log_object_lock_acquire_if_slow(
1599 op: &'static str,
1600 bucket: &str,
1601 object: &str,
1602 owner: Option<&str>,
1603 mode: ObjectLockDiagMode,
1604 elapsed: Duration,
1605 diag_enabled: bool,
1606) {
1607 if !diag_enabled {
1608 return;
1609 }
1610
1611 let threshold = get_object_lock_diag_slow_acquire_threshold();
1612 record_object_lock_diag_acquire_duration(op, mode.as_str(), elapsed);
1613 if elapsed >= threshold {
1614 record_object_lock_diag_slow_acquire(op, mode.as_str());
1615 warn!(
1616 target: "rustfs_ecstore::object_lock_diag",
1617 op,
1618 bucket,
1619 object,
1620 mode = %mode,
1621 owner = owner.unwrap_or_default(),
1622 acquire_ms = elapsed.as_millis(),
1623 threshold_ms = threshold.as_millis(),
1624 "object namespace lock acquisition exceeded threshold"
1625 );
1626 }
1627}
1628
1629fn select_data_movement_target_pool(
1630 existing_pool_idx: Result<usize>,
1631 src_pool_idx: usize,
1632 delete_marker: bool,
1633) -> Result<Option<usize>> {
1634 match existing_pool_idx {
1635 Ok(pool_idx) => {
1636 if delete_marker && pool_idx == src_pool_idx {
1637 Ok(None)
1638 } else {
1639 Ok(Some(pool_idx))
1640 }
1641 }
1642 Err(err) => {
1643 if is_err_read_quorum(&err) {
1644 return Err(StorageError::ErasureWriteQuorum);
1645 }
1646 if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
1647 Ok(None)
1648 } else {
1649 Err(err)
1650 }
1651 }
1652 }
1653}
1654
1655fn latest_object_access_delete_marker_error(
1656 bucket: &str,
1657 object: &str,
1658 info: &ObjectInfo,
1659 opts: &ObjectOptions,
1660) -> Option<Error> {
1661 if !info.delete_marker {
1662 return None;
1663 }
1664
1665 Some(if opts.version_id.is_none() || opts.delete_marker {
1666 to_object_err(StorageError::FileNotFound, vec![bucket, object])
1667 } else {
1668 to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])
1669 })
1670}
1671
1672fn resolve_latest_object_access(
1673 bucket: &str,
1674 object: &str,
1675 info: ObjectInfo,
1676 idx: usize,
1677 opts: &ObjectOptions,
1678) -> Result<(ObjectInfo, usize)> {
1679 if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) {
1680 return Err(err);
1681 }
1682
1683 Ok((info, idx))
1684}
1685
1686fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
1687 (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
1688}
1689
1690fn latest_versioned_delete_creates_distinct_marker(opts: &ObjectOptions) -> bool {
1691 opts.versioned
1692 && !opts.version_suspended
1693 && opts.version_id.is_none()
1694 && !opts.delete_marker
1695 && !opts.data_movement
1696 && !opts.replication_request
1697}
1698
1699#[cfg(any(test, feature = "test-util"))]
1700struct DeleteAfterObjectLockSnapshotBarrierState {
1701 bucket: String,
1702 arrived: tokio::sync::Notify,
1703 release: tokio::sync::Notify,
1704 namespace_pending: tokio::sync::Notify,
1705 namespace_acquired: AtomicBool,
1706}
1707
1708#[cfg(any(test, feature = "test-util"))]
1709pub struct DeleteAfterObjectLockSnapshotBarrier {
1710 state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
1711}
1712
1713#[cfg(any(test, feature = "test-util"))]
1714static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
1715 std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
1716> = std::sync::OnceLock::new();
1717
1718#[cfg(any(test, feature = "test-util"))]
1719impl DeleteAfterObjectLockSnapshotBarrier {
1720 pub fn install(bucket: &str) -> Self {
1721 let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
1722 bucket: bucket.to_string(),
1723 arrived: tokio::sync::Notify::new(),
1724 release: tokio::sync::Notify::new(),
1725 namespace_pending: tokio::sync::Notify::new(),
1726 namespace_acquired: AtomicBool::new(false),
1727 });
1728 let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1729 .get_or_init(|| std::sync::Mutex::new(None))
1730 .lock()
1731 .expect("delete snapshot barrier mutex should not poison");
1732 assert!(slot.is_none(), "delete snapshot barrier must not already be installed");
1733 *slot = Some(Arc::clone(&state));
1734 Self { state }
1735 }
1736
1737 pub async fn wait_until_paused(&self) {
1738 self.state.arrived.notified().await;
1739 }
1740
1741 pub fn release(&self) {
1742 self.state.release.notify_one();
1743 }
1744
1745 pub async fn release_and_wait_until_namespace_pending(&self) {
1746 let namespace_pending = self.state.namespace_pending.notified();
1747 self.release();
1748 tokio::time::timeout(Duration::from_secs(5), namespace_pending)
1749 .await
1750 .expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
1751 }
1752
1753 pub fn namespace_acquired(&self) -> bool {
1754 self.state.namespace_acquired.load(Ordering::Acquire)
1755 }
1756}
1757
1758#[cfg(any(test, feature = "test-util"))]
1759impl Drop for DeleteAfterObjectLockSnapshotBarrier {
1760 fn drop(&mut self) {
1761 self.state.release.notify_one();
1762 if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() {
1763 let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison");
1764 if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
1765 *slot = None;
1766 }
1767 }
1768 }
1769}
1770
1771#[cfg(any(test, feature = "test-util"))]
1772async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
1773 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1774 .get_or_init(|| std::sync::Mutex::new(None))
1775 .lock()
1776 .expect("delete snapshot barrier mutex should not poison")
1777 .as_ref()
1778 .filter(|state| state.bucket == bucket)
1779 .cloned();
1780 if let Some(state) = state {
1781 state.arrived.notify_one();
1782 state.release.notified().await;
1783 }
1784}
1785
1786#[cfg(any(test, feature = "test-util"))]
1787fn notify_delete_namespace_pending(bucket: &str) {
1788 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1789 .get_or_init(|| std::sync::Mutex::new(None))
1790 .lock()
1791 .expect("delete snapshot barrier mutex should not poison")
1792 .as_ref()
1793 .filter(|state| state.bucket == bucket)
1794 .cloned();
1795 if let Some(state) = state {
1796 state.namespace_pending.notify_one();
1797 }
1798}
1799
1800#[cfg(any(test, feature = "test-util"))]
1801fn notify_delete_namespace_acquired(bucket: &str) {
1802 let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1803 .get_or_init(|| std::sync::Mutex::new(None))
1804 .lock()
1805 .expect("delete snapshot barrier mutex should not poison")
1806 .as_ref()
1807 .filter(|state| state.bucket == bucket)
1808 .cloned();
1809 if let Some(state) = state {
1810 state.namespace_acquired.store(true, Ordering::Release);
1811 }
1812}
1813
1814#[cfg(test)]
1815struct VersionedDeleteMarkerCommitBarrierState {
1816 bucket: String,
1817 object: String,
1818 arrived: tokio::sync::Notify,
1819 release: tokio::sync::Notify,
1820}
1821
1822#[cfg(test)]
1823pub(crate) struct VersionedDeleteMarkerCommitBarrier {
1824 state: Arc<VersionedDeleteMarkerCommitBarrierState>,
1825}
1826
1827#[cfg(test)]
1828static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock<
1829 std::sync::Mutex<Option<Arc<VersionedDeleteMarkerCommitBarrierState>>>,
1830> = std::sync::OnceLock::new();
1831
1832#[cfg(test)]
1833impl VersionedDeleteMarkerCommitBarrier {
1834 pub(crate) fn install(bucket: &str, object: &str) -> Self {
1835 let state = Arc::new(VersionedDeleteMarkerCommitBarrierState {
1836 bucket: bucket.to_string(),
1837 object: object.to_string(),
1838 arrived: tokio::sync::Notify::new(),
1839 release: tokio::sync::Notify::new(),
1840 });
1841 let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1842 .get_or_init(|| std::sync::Mutex::new(None))
1843 .lock()
1844 .expect("versioned delete-marker commit barrier mutex should not poison");
1845 assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique");
1846 *slot = Some(Arc::clone(&state));
1847 Self { state }
1848 }
1849
1850 pub(crate) async fn wait_until_paused(&self) {
1851 tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1852 .await
1853 .expect("versioned DELETE should reach the post-marker-commit barrier");
1854 }
1855
1856 pub(crate) fn release(&self) {
1857 self.state.release.notify_one();
1858 }
1859}
1860
1861#[cfg(test)]
1862impl Drop for VersionedDeleteMarkerCommitBarrier {
1863 fn drop(&mut self) {
1864 self.state.release.notify_one();
1865 let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1866 .get_or_init(|| std::sync::Mutex::new(None))
1867 .lock()
1868 .expect("versioned delete-marker commit barrier mutex should not poison");
1869 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1870 *slot = None;
1871 }
1872 }
1873}
1874
1875#[cfg(test)]
1876async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) {
1877 let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1878 .get_or_init(|| std::sync::Mutex::new(None))
1879 .lock()
1880 .expect("versioned delete-marker commit barrier mutex should not poison")
1881 .as_ref()
1882 .filter(|state| state.bucket == bucket && state.object == object)
1883 .cloned();
1884 if let Some(state) = state {
1885 state.arrived.notify_one();
1886 state.release.notified().await;
1887 }
1888}
1889
1890fn should_purge_orphan_dir_on_missing(err: &Error, object: &str) -> bool {
1899 (is_err_object_not_found(err) || is_err_version_not_found(err)) && rustfs_utils::path::is_dir_object(object)
1900}
1901
1902fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1903 let mut lookup_opts = opts.clone();
1904 lookup_opts.no_lock = no_lock;
1905 if lookup_opts.version_id.is_some() {
1906 lookup_opts.metadata_chg = true;
1907 }
1908
1909 lookup_opts
1910}
1911
1912fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1913 writer_pool_lookup_opts(opts, no_lock)
1914}
1915
1916fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool {
1917 opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some())
1918}
1919
1920fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1921 let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
1922 lookup_opts.skip_decommissioned = true;
1923 lookup_opts.skip_rebalancing = true;
1924
1925 lookup_opts
1926}
1927
1928fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1929 let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock);
1930 lookup_opts.skip_decommissioned = opts.data_movement;
1931 lookup_opts
1932}
1933
1934fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
1935 pool_count > 0 && delete_only_releases_capacity(opts)
1936}
1937
1938fn delete_only_releases_capacity(opts: &ObjectOptions) -> bool {
1939 !opts.versioned && !opts.version_suspended || opts.version_id.is_some()
1940}
1941
1942fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
1943 if object.version_id.is_some() {
1944 return false;
1945 }
1946
1947 let object_name = decode_dir_object(&object.object_name);
1948 let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name);
1949 versioned || version_suspended
1950}
1951
1952fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option<usize>, pool_idx: usize) -> bool {
1953 !creates_latest_marker || marker_target_pool_idx == Some(pool_idx)
1954}
1955
1956#[cfg(test)]
1957struct BatchDeletePoolErrorInjectionState {
1958 bucket: String,
1959 pool_idx: usize,
1960 errors: std::collections::HashMap<String, Error>,
1961 observed: std::sync::atomic::AtomicUsize,
1962}
1963
1964#[cfg(test)]
1965pub(crate) struct BatchDeletePoolErrorInjection {
1966 state: Arc<BatchDeletePoolErrorInjectionState>,
1967}
1968
1969#[cfg(test)]
1970static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock<std::sync::Mutex<Option<Arc<BatchDeletePoolErrorInjectionState>>>> =
1971 std::sync::OnceLock::new();
1972
1973#[cfg(test)]
1974impl BatchDeletePoolErrorInjection {
1975 pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self {
1976 let state = Arc::new(BatchDeletePoolErrorInjectionState {
1977 bucket: bucket.to_string(),
1978 pool_idx,
1979 errors: errors.into_iter().collect(),
1980 observed: std::sync::atomic::AtomicUsize::new(0),
1981 });
1982 let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
1983 .get_or_init(|| std::sync::Mutex::new(None))
1984 .lock()
1985 .expect("batch delete pool error injection mutex should not poison");
1986 assert!(slot.is_none(), "batch delete pool error injection must be unique");
1987 *slot = Some(Arc::clone(&state));
1988 Self { state }
1989 }
1990
1991 pub(crate) fn observed(&self) -> usize {
1992 self.state.observed.load(Ordering::Acquire)
1993 }
1994}
1995
1996#[cfg(test)]
1997impl Drop for BatchDeletePoolErrorInjection {
1998 fn drop(&mut self) {
1999 let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
2000 .get_or_init(|| std::sync::Mutex::new(None))
2001 .lock()
2002 .expect("batch delete pool error injection mutex should not poison");
2003 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
2004 *slot = None;
2005 }
2006 }
2007}
2008
2009#[cfg(test)]
2010fn inject_batch_delete_pool_errors(
2011 bucket: &str,
2012 pool_idx: usize,
2013 object_names: &[String],
2014 deleted: &[DeletedObject],
2015 errors: &mut [Option<Error>],
2016) {
2017 let state = BATCH_DELETE_POOL_ERROR_INJECTION
2018 .get_or_init(|| std::sync::Mutex::new(None))
2019 .lock()
2020 .expect("batch delete pool error injection mutex should not poison")
2021 .as_ref()
2022 .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx)
2023 .cloned();
2024 let Some(state) = state else {
2025 return;
2026 };
2027
2028 for (idx, object_name) in object_names.iter().enumerate() {
2029 let Some(error) = state.errors.get(object_name) else {
2030 continue;
2031 };
2032 if errors[idx].is_none() && deleted[idx].found {
2033 errors[idx] = Some(error.clone());
2034 state.observed.fetch_add(1, Ordering::AcqRel);
2035 }
2036 }
2037}
2038
2039fn resolve_batch_delete_pool_results<'a>(
2040 initial_error: Option<Error>,
2041 pool_results: impl IntoIterator<Item = (&'a DeletedObject, &'a Option<Error>)>,
2042) -> (Option<DeletedObject>, Option<Error>, bool) {
2043 let mut failure = initial_error.map(|err| (None, err));
2044 let mut deleted = None;
2045 let mut fallback: Option<(DeletedObject, Option<Error>)> = None;
2046 let mut attempted = false;
2047
2048 for (pool_delete, pool_error) in pool_results {
2049 attempted = true;
2050 match pool_error {
2051 Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => {
2052 if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) {
2053 fallback = Some(((*pool_delete).clone(), Some(err.clone())));
2054 }
2055 }
2056 Some(err) => {
2057 if failure.is_none() {
2058 failure = Some((Some((*pool_delete).clone()), err.clone()));
2059 }
2060 }
2061 None if pool_delete.found => {
2062 if deleted.is_none() {
2063 deleted = Some((*pool_delete).clone());
2064 }
2065 }
2066 None => {
2067 if fallback.is_none() {
2068 fallback = Some(((*pool_delete).clone(), None));
2069 }
2070 }
2071 }
2072 }
2073
2074 if let Some((failed_delete, err)) = failure {
2075 return (failed_delete, Some(err), attempted);
2076 }
2077 if let Some(deleted) = deleted {
2078 return (Some(deleted), None, attempted);
2079 }
2080 if let Some((deleted, err)) = fallback {
2081 return (Some(deleted), err, attempted);
2082 }
2083
2084 (None, None, attempted)
2085}
2086
2087fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
2088 let mut lookup_opts = opts.clone();
2089 lookup_opts.no_lock = false;
2093 lookup_opts.skip_decommissioned = true;
2094 lookup_opts.skip_rebalancing = true;
2095 lookup_opts
2096}
2097
2098fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
2099 info.get_actual_size().ok()
2100}
2101
2102pub(super) fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2103 is_data_movement_delete_marker(source)
2104 && is_data_movement_delete_marker(target)
2105 && source.version_id == target.version_id
2106 && source.mod_time == target.mod_time
2107 && is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
2108 && source.user_tags == target.user_tags
2109 && source.replication_status_internal == target.replication_status_internal
2110 && source.replication_status == target.replication_status
2111 && source.version_purge_status_internal == target.version_purge_status_internal
2112 && source.version_purge_status == target.version_purge_status
2113}
2114
2115fn is_equivalent_data_movement_delete_marker_metadata(
2116 source: &HashMap<String, String>,
2117 target: &HashMap<String, String>,
2118) -> bool {
2119 matches!(
2120 (
2121 data_movement_delete_marker_metadata_identity(source),
2122 data_movement_delete_marker_metadata_identity(target)
2123 ),
2124 (Some(source), Some(target)) if source == target
2125 )
2126}
2127
2128fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
2129 let mut identity = HashMap::with_capacity(metadata.len());
2130 let mut local_tier_free_version_id = None;
2131 for (key, value) in metadata {
2132 let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
2133 identity.insert(key.clone(), value.clone());
2134 continue;
2135 };
2136
2137 if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
2138 let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
2139 if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
2140 return None;
2141 }
2142 local_tier_free_version_id = Some(version_id);
2143 continue;
2144 }
2145 if suffix.eq_ignore_ascii_case(rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID) {
2146 continue;
2147 }
2148
2149 let canonical_suffix = [
2150 rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
2151 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
2152 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2153 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2154 rustfs_utils::http::SUFFIX_PURGESTATUS,
2155 ]
2156 .into_iter()
2157 .find(|candidate| suffix.eq_ignore_ascii_case(candidate))
2158 .map(str::to_string)
2159 .or_else(|| {
2160 [
2161 rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
2162 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
2163 ]
2164 .into_iter()
2165 .find_map(|prefix| {
2166 suffix
2167 .get(..prefix.len())
2168 .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
2169 .then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
2170 })
2171 })
2172 .unwrap_or_else(|| suffix.to_string());
2173 let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
2174 || canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
2175 {
2176 rustfs_filemeta::parse_replication_timestamp(value)?
2177 .unix_timestamp_nanos()
2178 .to_string()
2179 } else {
2180 value.clone()
2181 };
2182 let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2183 if identity
2184 .insert(canonical_key, canonical_value.clone())
2185 .is_some_and(|existing| existing != canonical_value)
2186 {
2187 return None;
2188 }
2189 }
2190 for (status_suffix, timestamp_suffix) in [
2191 (rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
2192 (
2193 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2194 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2195 ),
2196 ] {
2197 let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2198 let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2199 match (identity.contains_key(&status_key), identity.contains_key(×tamp_key)) {
2200 (true, false) => {
2201 identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
2202 }
2203 (false, true) => return None,
2204 _ => {}
2205 }
2206 }
2207 Some(identity)
2208}
2209
2210fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
2211 info.delete_marker
2212}
2213
2214fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
2215 is_data_movement_delete_marker(source)
2216 && source.mod_time.is_some()
2217 && source.mod_time == expected_mod_time
2218 && data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
2219}
2220
2221fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
2222 let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
2223 let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
2224 let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
2225 (None, None) => Default::default(),
2226 (Some(status), timestamp) => {
2227 let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2228 if status.is_empty() {
2229 return None;
2230 }
2231 let timestamp = match timestamp {
2232 Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
2233 None => OffsetDateTime::UNIX_EPOCH,
2234 };
2235 (status, Some(timestamp))
2236 }
2237 (None, Some(_)) => return None,
2238 };
2239 let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
2240 let replication_timestamp =
2241 rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
2242 let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
2243 (None, None) => Default::default(),
2244 (Some(status), timestamp) => {
2245 let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2246 let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
2247 if direct_status.is_empty() && targets.is_empty() {
2248 return None;
2249 }
2250 let timestamp = match timestamp {
2251 Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
2252 None => OffsetDateTime::UNIX_EPOCH,
2253 };
2254 (Some(status), Some(timestamp), targets)
2255 }
2256 (None, Some(_)) => return None,
2257 };
2258 let mut state = source.replication_state();
2259 if state.target_delete_marker_version_ids_corrupt {
2260 return None;
2261 }
2262 state.replica_status = replica_status;
2263 state.replica_timestamp = replica_timestamp;
2264 state.replication_status_internal = replication_status;
2265 state.replication_timestamp = replication_timestamp;
2266 state.targets = replication_targets;
2267 state.replicate_decision_str = source.replication_decision.clone();
2268 state.delete_marker = true;
2269
2270 let mut target_opts = opts.clone();
2271 target_opts.mod_time = source.mod_time;
2272 target_opts.delete_replication = Some(state);
2273 Some(target_opts)
2274}
2275
2276fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
2277 ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
2278}
2279
2280fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
2281 let expected = expected_data_movement_tiered_object(source);
2282 let Some(source_actual_size) = effective_object_actual_size(&expected) else {
2283 return false;
2284 };
2285 let Some(target_actual_size) = effective_object_actual_size(target) else {
2286 return false;
2287 };
2288
2289 source.version_id == target.version_id
2290 && !target.delete_marker
2291 && source.size == target.size
2292 && source.get_etag() == target.etag
2293 && source.checksum == target.checksum
2294 && crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
2295 && source.mod_time == target.mod_time
2296 && crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
2297 && expected.user_tags == target.user_tags
2298 && expected.expires == target.expires
2299 && expected.storage_class == target.storage_class
2300 && expected.replication_status_internal == target.replication_status_internal
2301 && expected.replication_status == target.replication_status
2302 && expected.version_purge_status_internal == target.version_purge_status_internal
2303 && expected.version_purge_status == target.version_purge_status
2304 && expected.transitioned_object.status == target.transitioned_object.status
2305 && expected.transition_version_state == target.transition_version_state
2306 && expected.transitioned_object.name == target.transitioned_object.name
2307 && expected.transitioned_object.tier == target.transitioned_object.tier
2308 && expected.transitioned_object.version_id == target.transitioned_object.version_id
2309 && expected.transitioned_object.free_version == target.transitioned_object.free_version
2310 && source_actual_size == target_actual_size
2311}
2312
2313pub(crate) fn tiered_data_movement_source_matches(
2314 expected: &rustfs_filemeta::FileInfo,
2315 current: &rustfs_filemeta::FileInfo,
2316) -> Result<bool> {
2317 let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
2318 let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
2319 Ok(expected.version_id == current.version_id
2320 && expected.deleted == current.deleted
2321 && expected.tier_free_version() == current.tier_free_version()
2322 && expected.data_dir == current.data_dir
2323 && expected.mod_time == current.mod_time
2324 && expected.size == current.size
2325 && expected.get_etag() == current.get_etag()
2326 && expected.transition_status == current.transition_status
2327 && expected.transitioned_objname == current.transitioned_objname
2328 && expected.transition_tier == current.transition_tier
2329 && expected.transition_version_id == current.transition_version_id
2330 && expected.transition_version == current.transition_version
2331 && expected.transition_version_state == current.transition_version_state
2332 && expected_backend == current_backend)
2333}
2334
2335fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
2336 StorageError::DataMovementOverwriteErr(
2337 bucket.to_owned(),
2338 object.to_owned(),
2339 version_id.map(|id| id.to_string()).unwrap_or_default(),
2340 )
2341}
2342
2343fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
2344 target_pool_idx != src_pool_idx
2345}
2346
2347fn resolve_data_movement_resume_target_pool(
2348 selected_target_pool_idx: usize,
2349 resume_target_pool_idx: Option<usize>,
2350 src_pool_idx: usize,
2351) -> usize {
2352 if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
2353 selected_target_pool_idx
2354 } else {
2355 resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
2356 }
2357}
2358
2359fn resolve_data_movement_delete_marker_resume_result(
2360 target_result: Result<Option<ObjectInfo>>,
2361 source: &ObjectInfo,
2362 src_pool_idx: usize,
2363 target_pool_idx: usize,
2364) -> Result<bool> {
2365 if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2366 return Ok(false);
2367 }
2368
2369 let Some(target) = target_result? else {
2370 return Ok(false);
2371 };
2372
2373 Ok(is_equivalent_data_movement_delete_marker(source, &target))
2374}
2375
2376fn resolve_data_movement_tiered_resume_result(
2377 target_result: Result<Option<ObjectInfo>>,
2378 source: &rustfs_filemeta::FileInfo,
2379 src_pool_idx: usize,
2380 target_pool_idx: usize,
2381) -> Result<bool> {
2382 if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2383 return Ok(false);
2384 }
2385
2386 let Some(target) = target_result? else {
2387 return Ok(false);
2388 };
2389
2390 Ok(is_equivalent_data_movement_tiered_object(source, &target))
2391}
2392
2393fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
2394 let del_objects = objects
2395 .iter()
2396 .map(|object| DeletedObject {
2397 object_name: decode_dir_object(&object.object_name),
2398 version_id: object.version_id,
2399 ..Default::default()
2400 })
2401 .collect();
2402 let del_errs = objects.iter().map(|_| Some(err.clone())).collect();
2403
2404 (del_objects, del_errs)
2405}
2406
2407fn return_batch_delete_lock_error_with_accounting(
2408 objects: &[ObjectToDelete],
2409 err: Error,
2410) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
2411 let (deleted, errors) = return_batch_delete_lock_error(objects, err);
2412 (deleted, errors, vec![None; objects.len()])
2413}
2414
2415fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
2416 let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
2417 object_names.sort_unstable();
2418 object_names.dedup();
2419 object_names
2420}
2421
2422fn materialized_copy_user_defined(source: &HashMap<String, String>) -> HashMap<String, String> {
2433 let mut metadata = source.clone();
2434 for suffix in [
2435 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2436 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2437 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2438 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2439 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2440 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2441 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2442 rustfs_utils::http::SUFFIX_FREE_VERSION,
2443 rustfs_utils::http::SUFFIX_TIER_FV_ID,
2444 rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
2445 ] {
2446 rustfs_utils::http::metadata_compat::remove_str(&mut metadata, suffix);
2447 }
2448 metadata
2449}
2450
2451enum RemoteTuplePublicationSource {
2452 Object(ObjectInfo),
2453 Tiered(rustfs_filemeta::FileInfo),
2454}
2455
2456#[must_use = "the fixed read anchor must remain live until publication or migration abort"]
2461pub(crate) struct DecommissionFixedReadAnchor {
2462 guard: ObjectLockDiagGuard,
2463}
2464
2465impl DecommissionFixedReadAnchor {
2466 pub(crate) fn guard(&self) -> &ObjectLockDiagGuard {
2467 &self.guard
2468 }
2469
2470 fn is_lock_lost(&self) -> bool {
2471 self.guard.is_lock_lost()
2472 }
2473}
2474
2475#[must_use = "a data-movement publication fence must be consumed by the final target commit"]
2484pub(crate) struct RemoteTuplePublicationFence {
2485 store: Arc<ECStore>,
2486 bucket: String,
2487 object: String,
2488 source_pool_idx: usize,
2489 source: RemoteTuplePublicationSource,
2490 include_fixed_domain: bool,
2491 fixed_read_anchor: Option<DecommissionFixedReadAnchor>,
2492 backend_target: Option<(String, Option<TierDestinationId>)>,
2493}
2494
2495#[must_use = "the publication commit guard must live through the target metadata commit"]
2498pub(crate) struct RemoteTuplePublicationCommitGuard {
2499 guards: Vec<ObjectLockDiagGuard>,
2500 backend_lease: Option<TierOperationLease>,
2501 revalidated_tiered_source: Option<rustfs_filemeta::FileInfo>,
2502}
2503
2504impl RemoteTuplePublicationCommitGuard {
2505 pub(crate) fn is_lock_lost(&self) -> bool {
2506 self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2507 || self
2508 .backend_lease
2509 .as_ref()
2510 .is_some_and(|lease| !lease.is_current_generation())
2511 }
2512
2513 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
2514 for guard in &self.guards {
2515 guard.add_namespace_lock_fence(opts);
2516 }
2517 }
2518
2519 pub(crate) fn revalidated_tiered_source(&self) -> Option<&rustfs_filemeta::FileInfo> {
2520 self.revalidated_tiered_source.as_ref()
2521 }
2522}
2523
2524fn remote_tuple_publication_target(source: &ObjectInfo) -> std::io::Result<Option<(String, Option<TierDestinationId>)>> {
2525 if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !source.transitioned_object.free_version {
2526 return Ok(None);
2527 }
2528 if source.transitioned_object.tier.is_empty() || source.transitioned_object.name.is_empty() {
2529 return Err(std::io::Error::other(
2530 "remote tuple publication source is missing its tier or remote object identity",
2531 ));
2532 }
2533 let backend_identity = tier_destination_id_from_metadata(source.user_defined.as_ref())?;
2534 Ok(Some((source.transitioned_object.tier.clone(), backend_identity)))
2535}
2536
2537fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current: &ObjectInfo) -> bool {
2538 let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else {
2539 return false;
2540 };
2541
2542 expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil())
2543 && expected.data_dir == current.data_dir
2544 && expected.version_id == current.version_id
2545 && expected.delete_marker == current.delete_marker
2546 && expected.size == current.size
2547 && expected_actual_size == current_actual_size
2548 && expected.etag == current.etag
2549 && expected.checksum == current.checksum
2550 && expected.mod_time == current.mod_time
2551 && expected.storage_class == current.storage_class
2552 && crate::data_movement::is_equivalent_data_movement_metadata(
2553 expected,
2554 current,
2555 expected_actual_size,
2556 current_actual_size,
2557 )
2558 && expected.user_tags == current.user_tags
2559 && expected.expires == current.expires
2560 && expected.replication_status_internal == current.replication_status_internal
2561 && expected.replication_status == current.replication_status
2562 && expected.version_purge_status_internal == current.version_purge_status_internal
2563 && expected.version_purge_status == current.version_purge_status
2564 && expected.transitioned_object.name == current.transitioned_object.name
2565 && expected.transitioned_object.version_id == current.transitioned_object.version_id
2566 && expected.transitioned_object.tier == current.transitioned_object.tier
2567 && expected.transitioned_object.free_version == current.transitioned_object.free_version
2568 && expected.transitioned_object.status == current.transitioned_object.status
2569 && expected.transition_version_state == current.transition_version_state
2570 && remote_tuple_publication_parts_match(&expected.parts, ¤t.parts)
2571}
2572
2573fn remote_tuple_publication_parts_match(expected: &[ObjectPartInfo], current: &[ObjectPartInfo]) -> bool {
2574 if expected.len() != current.len() {
2575 return false;
2576 }
2577 let Some(mut current_parts) = crate::data_movement::data_movement_parts_by_number(current) else {
2578 return false;
2579 };
2580
2581 expected.iter().all(|expected_part| {
2582 current_parts.remove(&expected_part.number).is_some_and(|current_part| {
2583 current_part.size == expected_part.size
2584 && current_part.actual_size == expected_part.actual_size
2585 && current_part.etag == expected_part.etag
2586 })
2587 })
2588}
2589
2590impl RemoteTuplePublicationFence {
2591 pub(crate) fn under_fixed_read_anchor(mut self, anchor: DecommissionFixedReadAnchor) -> Result<Self> {
2592 if self.include_fixed_domain {
2593 return Err(Error::other(
2594 "data movement publication capability cannot attach a fixed read anchor after requesting fixed write",
2595 ));
2596 }
2597 if anchor.is_lock_lost() {
2598 return Err(Error::other(
2599 "data movement publication capability received an already-lost fixed read anchor",
2600 ));
2601 }
2602 self.include_fixed_domain = self.backend_target.is_some();
2603 self.fixed_read_anchor = Some(anchor);
2604 Ok(self)
2605 }
2606
2607 pub(crate) fn fixed_read_anchor_guard(&self) -> Option<&ObjectLockDiagGuard> {
2608 self.fixed_read_anchor.as_ref().map(DecommissionFixedReadAnchor::guard)
2609 }
2610
2611 pub(crate) async fn into_commit_guard(
2612 self,
2613 target_pool_idx: usize,
2614 bucket: &str,
2615 object: &str,
2616 ) -> Result<RemoteTuplePublicationCommitGuard> {
2617 let Self {
2618 store,
2619 bucket: expected_bucket,
2620 object: expected_object,
2621 source_pool_idx,
2622 source,
2623 include_fixed_domain,
2624 fixed_read_anchor,
2625 backend_target,
2626 } = self;
2627 if bucket != expected_bucket || object != expected_object {
2628 return Err(Error::other_with_context(
2629 "data movement publication capability target mismatch",
2630 format!("expected {expected_bucket}/{expected_object}, got {bucket}/{object}"),
2631 ));
2632 }
2633 if fixed_read_anchor
2634 .as_ref()
2635 .is_some_and(DecommissionFixedReadAnchor::is_lock_lost)
2636 {
2637 return Err(Error::other("data movement publication fixed read anchor was lost before commit"));
2638 }
2639 drop(fixed_read_anchor);
2643 let backend_lease = if let Some((tier_name, backend_identity)) = backend_target {
2646 let manager = store.tier_config_mgr();
2647 Some(
2648 match backend_identity {
2649 Some(identity) => {
2650 TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, &tier_name, identity).await
2651 }
2652 None => TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await,
2653 }
2654 .map_err(Error::other)?,
2655 )
2656 } else {
2657 None
2658 };
2659
2660 let guards = store
2661 .acquire_data_movement_publication_write_locks(bucket, object, source_pool_idx, target_pool_idx, include_fixed_domain)
2662 .await?;
2663 if guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2664 || backend_lease.as_ref().is_some_and(|lease| !lease.is_current_generation())
2665 {
2666 return Err(Error::other("data movement publication fence was lost before source revalidation"));
2667 }
2668
2669 let source_pool = store.pools.get(source_pool_idx).ok_or_else(|| {
2670 Error::other_with_context("invalid data movement source pool", format!("pool index {source_pool_idx}"))
2671 })?;
2672 let mut revalidated_tiered_source = None;
2673 let source_matches = match source {
2674 RemoteTuplePublicationSource::Object(expected) => {
2675 let lookup_opts = ObjectOptions {
2676 versioned: expected.version_id.is_some(),
2677 version_id: expected.version_id.map(|version_id| version_id.to_string()),
2678 incl_free_versions: expected.transitioned_object.free_version,
2679 include_part_checksums: true,
2680 no_lock: true,
2681 ..Default::default()
2682 };
2683 match source_pool.get_object_info(bucket, object, &lookup_opts).await {
2684 Ok(current) => remote_tuple_publication_object_source_matches(&expected, ¤t),
2685 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => false,
2686 Err(err) => return Err(err),
2687 }
2688 }
2689 RemoteTuplePublicationSource::Tiered(expected) => {
2690 let logical_object = decode_dir_object(object);
2691 let current_versions = source_pool
2692 .get_disks_by_key(object)
2693 .load_file_info_versions_exact(bucket, &logical_object)
2694 .await?;
2695 let is_free_version = expected.tier_free_version();
2696 let Some(current) = current_versions.as_ref().and_then(|versions| {
2697 versions.versions.iter().find(|current| {
2698 current.version_id == expected.version_id && current.tier_free_version() == is_free_version
2699 })
2700 }) else {
2701 return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
2702 };
2703 if tiered_data_movement_source_matches(&expected, current)? {
2704 revalidated_tiered_source = Some(current.clone());
2705 true
2706 } else {
2707 false
2708 }
2709 }
2710 };
2711 if !source_matches {
2712 return Err(StorageError::DataMovementOverwriteErr(
2713 bucket.to_string(),
2714 object.to_string(),
2715 String::new(),
2716 ));
2717 }
2718
2719 let commit_guard = RemoteTuplePublicationCommitGuard {
2720 guards,
2721 backend_lease,
2722 revalidated_tiered_source,
2723 };
2724 if commit_guard.is_lock_lost() {
2725 return Err(Error::other("data movement publication fence was lost after source revalidation"));
2726 }
2727 Ok(commit_guard)
2728 }
2729}
2730
2731impl ECStore {
2732 pub(crate) async fn acquire_remote_tuple_publication_fence(
2737 self: &Arc<Self>,
2738 bucket: &str,
2739 source_pool_idx: usize,
2740 source: &ObjectInfo,
2741 include_fixed_domain: bool,
2742 ) -> Result<RemoteTuplePublicationFence> {
2743 if self.pools.get(source_pool_idx).is_none() {
2744 return Err(Error::other_with_context(
2745 "data movement source pool is out of range",
2746 format!("pool index {source_pool_idx} for {bucket}/{}", source.name),
2747 ));
2748 }
2749 let backend_target = remote_tuple_publication_target(source).map_err(Error::other)?;
2750 let include_fixed_domain = include_fixed_domain && backend_target.is_some();
2751 Ok(RemoteTuplePublicationFence {
2752 store: Arc::clone(self),
2753 bucket: bucket.to_string(),
2754 object: encode_dir_object(&source.name),
2755 source_pool_idx,
2756 source: RemoteTuplePublicationSource::Object(source.clone()),
2757 include_fixed_domain,
2758 fixed_read_anchor: None,
2759 backend_target,
2760 })
2761 }
2762
2763 async fn acquire_tiered_remote_tuple_publication_fence(
2764 self: &Arc<Self>,
2765 bucket: &str,
2766 object: &str,
2767 source_pool_idx: usize,
2768 source: &rustfs_filemeta::FileInfo,
2769 ) -> Result<RemoteTuplePublicationFence> {
2770 let object_info = ObjectInfo::from_file_info(source, bucket, object, true);
2771 let Some((tier_name, backend_identity)) = remote_tuple_publication_target(&object_info).map_err(Error::other)? else {
2772 return Err(Error::other("tiered data movement source has no publishable remote tuple"));
2773 };
2774 Ok(RemoteTuplePublicationFence {
2775 store: Arc::clone(self),
2776 bucket: bucket.to_string(),
2777 object: encode_dir_object(object),
2778 source_pool_idx,
2779 source: RemoteTuplePublicationSource::Tiered(source.clone()),
2780 include_fixed_domain: true,
2781 fixed_read_anchor: None,
2782 backend_target: Some((tier_name, backend_identity)),
2783 })
2784 }
2785
2786 pub(crate) fn is_equivalent_decommission_capacity_target(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2787 source.bucket == target.bucket
2788 && source.name == decode_dir_object(&target.name)
2789 && if source.delete_marker {
2790 is_equivalent_data_movement_delete_marker(source, target)
2791 } else {
2792 crate::data_movement::is_equivalent_data_movement_object_identity(source, target, true, false)
2793 }
2794 }
2795
2796 pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
2800 check_valid_bucket_name(bucket)?;
2801 let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
2802 let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2803 let (state, bucket_incarnation_id, config_revision) =
2804 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2805 if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
2806 return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
2807 }
2808 Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket(
2809 self.id,
2810 bucket,
2811 bucket_incarnation_id,
2812 config_revision,
2813 state,
2814 lifecycle_guard,
2815 metadata_guard,
2816 )))
2817 }
2818
2819 pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
2820 &self,
2821 bucket: &str,
2822 lifecycle_fence: &NamespaceLockFence,
2823 ) -> Result<Arc<ObjectLockConfigSnapshot>> {
2824 if lifecycle_fence.is_lock_lost() {
2825 return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
2826 }
2827 let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2828 let (state, bucket_incarnation_id, config_revision) =
2829 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2830 if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
2831 return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
2832 }
2833 Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
2834 self.id,
2835 bucket,
2836 bucket_incarnation_id,
2837 config_revision,
2838 state,
2839 lifecycle_fence.clone(),
2840 Arc::new(metadata_guard),
2841 )))
2842 }
2843
2844 pub async fn prepare_select_object_snapshot(
2850 &self,
2851 bucket: &str,
2852 object: &str,
2853 headers: &HeaderMap,
2854 opts: &ObjectOptions,
2855 ) -> std::result::Result<SelectObjectSnapshot, PrepareSelectObjectSnapshotError> {
2856 check_get_obj_args(bucket, object)?;
2857
2858 let object = encode_dir_object(object);
2859 let mut opts = opts.clone();
2860 opts.no_lock = false;
2861 opts.metadata_cache_safe = false;
2862 let read_lock_guards = self
2863 .acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
2864 .await?;
2865 if self.ctx.lock_manager().is_disabled() {
2866 return Err(SnapshotConsistencyError::LockingDisabled.into());
2867 }
2868 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2869 return Err(SnapshotConsistencyError::LockLost.into());
2870 }
2871
2872 let (mut metadata, pool) = if self.single_pool() {
2873 let pool = Arc::clone(&self.pools[0]);
2874 let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2875 (metadata, pool)
2876 } else {
2877 Box::pin(async {
2880 let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2881 let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
2882 StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
2883 })?;
2884 Ok::<_, StorageError>((metadata, pool))
2885 })
2886 .await?
2887 };
2888 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2889 return Err(SnapshotConsistencyError::LockLost.into());
2890 }
2891 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2892 return Err(error.into());
2893 }
2894
2895 let logical_size_i64 = metadata.object_info().get_actual_size().map_err(StorageError::from)?;
2896 let logical_size = u64::try_from(logical_size_i64)
2897 .map_err(|_| PrepareSelectObjectSnapshotError::InvalidLogicalSize { size: logical_size_i64 })?;
2898 let read_semantics_identity = metadata.read_semantics_identity();
2899 let object_info = metadata.take_object_info();
2900 if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2901 return Err(SnapshotConsistencyError::LockLost.into());
2902 }
2903
2904 Ok(SelectObjectSnapshot {
2905 pool,
2906 bucket: bucket.to_owned(),
2907 object,
2908 headers: rustfs_utils::http::project_ssec_transport_headers(headers),
2909 opts,
2910 object_info,
2911 logical_size,
2912 read_semantics_identity,
2913 first_metadata: ParkingMutex::new(Some(metadata)),
2914 lease: Arc::new(SelectObjectSnapshotLease::new(read_lock_guards)),
2915 })
2916 }
2917
2918 pub async fn prepare_get_object_reader(
2919 &self,
2920 bucket: &str,
2921 object: &str,
2922 range: Option<HTTPRangeSpec>,
2923 headers: HeaderMap,
2924 opts: &ObjectOptions,
2925 ) -> Result<PreparedGetObjectReader> {
2926 check_get_obj_args(bucket, object)?;
2927
2928 let object = encode_dir_object(object);
2929 let mut opts = opts.clone();
2930 let read_lock_guard = self
2931 .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
2932 .await?;
2933
2934 let (metadata, pool) = if self.single_pool() {
2935 let pool = Arc::clone(&self.pools[0]);
2936 let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2937 (metadata, pool)
2938 } else {
2939 Box::pin(async {
2942 let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2943 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2944 return Err(error);
2945 }
2946 let pool = self
2947 .pools
2948 .get(pool_idx)
2949 .cloned()
2950 .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
2951 Ok((metadata, pool))
2952 })
2953 .await?
2954 };
2955
2956 Ok(PreparedGetObjectReader {
2957 pool,
2958 bucket: bucket.to_owned(),
2959 object,
2960 range,
2961 headers,
2962 opts,
2963 metadata,
2964 read_lock_guard,
2965 })
2966 }
2967
2968 fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
2969 match err {
2970 rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
2971 mode,
2972 bucket: bucket.to_string(),
2973 object: object.to_string(),
2974 required,
2975 achieved,
2976 },
2977 other => StorageError::Lock(other),
2978 }
2979 }
2980
2981 pub(super) async fn acquire_object_write_lock(
2982 &self,
2983 op: &'static str,
2984 bucket: &str,
2985 object: &str,
2986 ) -> Result<ObjectLockDiagGuard> {
2987 let diag_enabled = is_object_lock_diag_enabled();
2988 let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
2989 let acquire_start = Instant::now();
2990 #[cfg(any(test, feature = "test-util"))]
2991 if matches!(op, "delete_object" | "delete_objects") {
2992 notify_delete_namespace_pending(bucket);
2993 }
2994 let timeout = get_lock_acquire_timeout();
2995 let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", timeout);
2996 let result = ns_lock.get_write_lock(timeout).await;
2997 attempt.observe(&result);
2998 let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
2999 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3000 log_object_lock_acquire_if_slow(
3001 op,
3002 bucket,
3003 object,
3004 owner.as_deref(),
3005 ObjectLockDiagMode::Write,
3006 acquire_start.elapsed(),
3007 diag_enabled,
3008 );
3009
3010 Ok(ObjectLockDiagGuard::new(
3011 guard,
3012 diag_enabled,
3013 op,
3014 diag_enabled.then(|| bucket.to_string()),
3015 diag_enabled.then(|| object.to_string()),
3016 owner,
3017 ObjectLockDiagMode::Write,
3018 )
3019 .with_attempt(attempt))
3020 }
3021
3022 async fn acquire_object_write_lock_if_needed(
3023 &self,
3024 op: &'static str,
3025 bucket: &str,
3026 object: &str,
3027 opts: &mut ObjectOptions,
3028 ) -> Result<Option<ObjectLockDiagGuard>> {
3029 if opts.no_lock {
3030 return Ok(None);
3031 }
3032
3033 let guard = self.acquire_object_write_lock(op, bucket, object).await?;
3034 if let Some(signal) = guard.lock_lost_signal() {
3035 opts.add_namespace_lock_lost_signal(signal);
3036 }
3037 opts.ensure_namespace_lock_fence();
3038 opts.no_lock = true;
3039
3040 Ok(Some(guard))
3041 }
3042
3043 pub async fn acquire_restore_accept_guard(&self, bucket: &str, object: &str) -> Result<RestoreAcceptGuard> {
3053 let object = encode_dir_object(object);
3054 let guard = self
3055 .acquire_object_write_lock("restore_object_accept", bucket, &object)
3056 .await?;
3057 Ok(RestoreAcceptGuard(guard))
3058 }
3059
3060 pub async fn acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<RestoreWorkerGuard> {
3065 let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3066 let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3067 lock.get_write_lock_quiet(get_lock_acquire_timeout())
3068 .await
3069 .map(RestoreWorkerGuard)
3070 .map_err(|err| Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err))
3071 }
3072
3073 pub async fn try_acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<Option<RestoreWorkerGuard>> {
3077 let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3078 let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3079 match lock.get_write_lock_quiet(RESTORE_WORKER_LOCK_PROBE_TIMEOUT).await {
3080 Ok(guard) => Ok(Some(RestoreWorkerGuard(guard))),
3081 Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => Ok(None),
3082 Err(err) => Err(Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err)),
3083 }
3084 }
3085
3086 async fn acquire_delete_objects_write_locks(
3087 &self,
3088 bucket: &str,
3089 objects: &[ObjectToDelete],
3090 opts: &mut ObjectOptions,
3091 ) -> Result<Vec<ObjectLockDiagGuard>> {
3092 if opts.no_lock || objects.is_empty() {
3093 return Ok(Vec::new());
3094 }
3095
3096 let object_names = sorted_unique_delete_object_names(objects);
3097 let mut guards = Vec::with_capacity(object_names.len());
3100 for object in object_names {
3101 guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
3102 }
3103 opts.no_lock = true;
3104 for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
3105 opts.add_namespace_lock_lost_signal(signal);
3106 }
3107 opts.ensure_namespace_lock_fence();
3108
3109 Ok(guards)
3110 }
3111
3112 async fn acquire_object_read_lock_if_needed(
3113 &self,
3114 op: &'static str,
3115 bucket: &str,
3116 object: &str,
3117 opts: &mut ObjectOptions,
3118 ) -> Result<Option<ObjectLockDiagGuard>> {
3119 if opts.no_lock {
3120 return Ok(None);
3121 }
3122
3123 let diag_enabled = is_object_lock_diag_enabled();
3124 let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
3125 let acquire_start = Instant::now();
3126 let timeout = get_lock_acquire_timeout();
3127 let mut attempt = ObjectLockAttempt::start(op, bucket, object, opts.version_id.as_deref(), &ns_lock, "read", timeout);
3128 let result = ns_lock.get_read_lock(timeout).await;
3129 attempt.observe(&result);
3130 let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3131 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3132 log_object_lock_acquire_if_slow(
3133 op,
3134 bucket,
3135 object,
3136 owner.as_deref(),
3137 ObjectLockDiagMode::Read,
3138 acquire_start.elapsed(),
3139 diag_enabled,
3140 );
3141 opts.no_lock = true;
3142 opts.metadata_cache_safe = true;
3143
3144 Ok(Some(
3145 ObjectLockDiagGuard::new(
3146 guard,
3147 diag_enabled,
3148 op,
3149 diag_enabled.then(|| bucket.to_string()),
3150 diag_enabled.then(|| object.to_string()),
3151 owner,
3152 ObjectLockDiagMode::Read,
3153 )
3154 .with_attempt(attempt),
3155 ))
3156 }
3157
3158 pub async fn save_scanner_pause_backlog_replica(
3162 self: &Arc<Self>,
3163 pool_index: usize,
3164 set_index: usize,
3165 data: Vec<u8>,
3166 preconditions: crate::storage_api_contracts::object::HTTPPreconditions,
3167 ) -> Result<ObjectInfo> {
3168 let set = self
3169 .pools
3170 .get(pool_index)
3171 .and_then(|pool| pool.disk_set.get(set_index))
3172 .ok_or_else(|| Error::InvalidArgument("scanner-backlog".into(), "replica".into(), "unknown pool or set".into()))?;
3173 let set = Arc::clone(set);
3174 let store = Arc::clone(self);
3175 let write = async move {
3176 let object = "buckets/.scanner-pause-backlog.json";
3177 let mut opts = ObjectOptions::default();
3178 let object_guard = if store.single_pool() {
3181 None
3182 } else {
3183 Some(
3184 store
3185 .acquire_object_write_lock("scanner_backlog_replica", RUSTFS_META_BUCKET, object)
3186 .await?,
3187 )
3188 };
3189 let capacity_guard = if let Some(guard) = object_guard.as_ref() {
3190 guard.add_namespace_lock_fence(&mut opts);
3191 opts.no_lock = match store.pools.first().and_then(|pool| pool.disk_set.first()) {
3192 Some(fixed) => fixed.shares_namespace_lock_domain(&set).await,
3193 None => false,
3194 };
3195 let capacity_guard = store
3196 .acquire_external_decommission_capacity_fence(&[pool_index], DecommissionCapacityAdmission::ScannerBacklog)
3197 .await?;
3198 opts.add_namespace_lock_guard(&capacity_guard);
3199 Some(capacity_guard)
3200 } else {
3201 None
3202 };
3203 let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
3204 set,
3205 data,
3206 preconditions,
3207 opts,
3208 "publish",
3209 )
3210 .await;
3211 drop(capacity_guard);
3212 drop(object_guard);
3213 result
3214 };
3215 tokio::spawn(write).await.map_err(Error::from)?
3218 }
3219
3220 pub(super) async fn run_external_decommission_capacity_object_mutation<T, F, Fut>(
3221 &self,
3222 target_pool_idx: usize,
3223 bucket: &str,
3224 lock_object: &str,
3225 target_object: &str,
3226 opts: ObjectOptions,
3227 operation: F,
3228 ) -> Result<T>
3229 where
3230 F: FnOnce(ObjectOptions) -> Fut,
3231 Fut: std::future::Future<Output = Result<T>>,
3232 {
3233 self.run_external_decommission_capacity_object_operation(
3234 target_pool_idx,
3235 bucket,
3236 (lock_object, target_object),
3237 opts,
3238 false,
3239 operation,
3240 )
3241 .await
3242 }
3243
3244 pub(super) async fn run_external_decommission_capacity_object_delete<T, F, Fut>(
3245 &self,
3246 target_pool_idx: usize,
3247 bucket: &str,
3248 lock_object: &str,
3249 target_object: &str,
3250 opts: ObjectOptions,
3251 operation: F,
3252 ) -> Result<T>
3253 where
3254 F: FnOnce(ObjectOptions) -> Fut,
3255 Fut: std::future::Future<Output = Result<T>>,
3256 {
3257 let capacity_releasing = delete_only_releases_capacity(&opts);
3258 self.run_external_decommission_capacity_object_operation(
3259 target_pool_idx,
3260 bucket,
3261 (lock_object, target_object),
3262 opts,
3263 capacity_releasing,
3264 operation,
3265 )
3266 .await
3267 }
3268
3269 async fn run_external_decommission_capacity_object_operation<T, F, Fut>(
3270 &self,
3271 target_pool_idx: usize,
3272 bucket: &str,
3273 objects: (&str, &str),
3274 mut opts: ObjectOptions,
3275 capacity_releasing: bool,
3276 operation: F,
3277 ) -> Result<T>
3278 where
3279 F: FnOnce(ObjectOptions) -> Fut,
3280 Fut: std::future::Future<Output = Result<T>>,
3281 {
3282 let (lock_object, target_object) = objects;
3283 if self.single_pool() {
3284 return operation(opts).await;
3285 }
3286 let (capacity_guard, has_active_decommission) = if capacity_releasing {
3287 self.acquire_decommission_capacity_release_fence_with_active_source().await?
3288 } else {
3289 self.acquire_external_decommission_capacity_fence_with_active_source(
3290 &[target_pool_idx],
3291 DecommissionCapacityAdmission::Mutation,
3292 )
3293 .await?
3294 };
3295 let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3296 drop(capacity_guard);
3299 #[cfg(test)]
3300 crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3301 let guard = self
3302 .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3303 .await?;
3304 self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
3305 .await;
3306 let capacity_guard = if capacity_releasing {
3307 self.acquire_decommission_capacity_release_fence_with_active_source().await?.0
3308 } else {
3309 self.acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Mutation)
3310 .await?
3311 };
3312 (capacity_guard, Some(guard))
3313 } else {
3314 (capacity_guard, None)
3315 };
3316
3317 let result = operation(opts).await;
3318 drop(capacity_guard);
3319 drop(object_guard);
3320 result
3321 }
3322
3323 pub(crate) async fn acquire_external_decommission_commit_guards(
3334 &self,
3335 target_pool_idx: usize,
3336 bucket: &str,
3337 object: &str,
3338 no_lock: bool,
3339 admission: DecommissionCapacityAdmission,
3340 ) -> Result<(Option<ObjectLockDiagGuard>, bool, Option<rustfs_lock::NamespaceLockGuard>)> {
3341 let (capacity_guard, has_active_decommission) = self
3342 .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], admission)
3343 .await?;
3344 if !has_active_decommission {
3345 #[cfg(test)]
3348 {
3349 crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id);
3350 crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await;
3351 }
3352 return Ok((None, false, Some(capacity_guard)));
3353 }
3354 drop(capacity_guard);
3355 if no_lock {
3356 return Ok((None, false, None));
3357 }
3358
3359 let object_guard = self
3364 .acquire_object_write_lock("external_capacity_order", bucket, object)
3365 .await?;
3366 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3367 let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3368 let target_lock_covered = match (fixed_set, target_set) {
3369 (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3370 _ => false,
3371 };
3372 Ok((Some(object_guard), target_lock_covered, None))
3373 }
3374
3375 pub(super) async fn run_external_decommission_capacity_heal<T, F, Fut>(
3376 &self,
3377 target_pool_idx: usize,
3378 bucket: &str,
3379 lock_object: &str,
3380 mut opts: HealOpts,
3381 operation: F,
3382 ) -> Result<T>
3383 where
3384 F: FnOnce(HealOpts) -> Fut,
3385 Fut: std::future::Future<Output = Result<T>>,
3386 {
3387 if self.single_pool() {
3388 return operation(opts).await;
3389 }
3390 let (capacity_guard, has_active_decommission) = self
3391 .acquire_external_decommission_capacity_fence_with_active_source(
3392 &[target_pool_idx],
3393 DecommissionCapacityAdmission::Heal,
3394 )
3395 .await?;
3396 let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3397 drop(capacity_guard);
3400 #[cfg(test)]
3401 crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3402 let guard = self
3403 .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3404 .await?;
3405 let target_set = self
3406 .pools
3407 .get(target_pool_idx)
3408 .ok_or_else(|| Error::other("heal target pool is unavailable"))?
3409 .get_disks_for_heal_object(lock_object, &opts)?;
3410 let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) {
3411 Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await,
3412 None => false,
3413 };
3414 let capacity_guard = self
3415 .acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Heal)
3416 .await?;
3417 opts.no_lock = target_lock_covered;
3418 (capacity_guard, Some(guard))
3419 } else {
3420 (capacity_guard, None)
3421 };
3422
3423 #[cfg(test)]
3424 if !opts.no_lock {
3425 crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
3426 }
3427 let result = operation(opts).await;
3428 drop(capacity_guard);
3429 drop(object_guard);
3430 result
3431 }
3432
3433 pub(crate) async fn acquire_decommission_object_mutation_fence(
3434 &self,
3435 bucket: &str,
3436 object: &str,
3437 ) -> Result<DecommissionFixedReadAnchor> {
3438 if self.ctx.lock_manager().is_disabled() {
3439 return Err(Error::other("decommission object migration requires namespace locking"));
3440 }
3441
3442 #[cfg(test)]
3443 let test_namespace_lock_fence =
3444 decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration);
3445 let object = encode_dir_object(object);
3446 let mut opts = ObjectOptions::default();
3447 let guard = self
3448 .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts)
3449 .await?
3450 .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?;
3451 #[cfg(test)]
3452 let guard = {
3453 let mut guard = guard;
3454 guard.test_namespace_lock_fence = test_namespace_lock_fence;
3455 guard
3456 };
3457 Ok(DecommissionFixedReadAnchor { guard })
3458 }
3459
3460 pub(super) async fn apply_decommission_target_mutation_fence(
3461 &self,
3462 target_pool_idx: usize,
3463 object: &str,
3464 opts: &mut ObjectOptions,
3465 mutation_fence: Option<&ObjectLockDiagGuard>,
3466 ) {
3467 let Some(mutation_fence) = mutation_fence else {
3468 return;
3469 };
3470
3471 mutation_fence.add_namespace_lock_fence(opts);
3472 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3473 let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3474 opts.no_lock = match (fixed_set, target_set) {
3475 (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3476 _ => false,
3477 };
3478 }
3479
3480 pub(crate) async fn acquire_decommission_source_cleanup_fence(
3481 &self,
3482 bucket: &str,
3483 object: &str,
3484 source_set: &SetDisks,
3485 ) -> Result<SourceCleanupMutationFence> {
3486 if self.ctx.lock_manager().is_disabled() {
3487 return Err(Error::other("decommission source cleanup requires namespace locking"));
3488 }
3489
3490 #[cfg(test)]
3491 crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object);
3492 #[cfg(test)]
3493 let test_namespace_lock_fence =
3494 decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup);
3495 let object = encode_dir_object(object);
3496 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3497 let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await;
3498 let guard = self
3501 .acquire_object_write_lock("decommission_source_cleanup", bucket, &object)
3502 .await?;
3503 #[cfg(test)]
3504 let guard = {
3505 let mut guard = guard;
3506 guard.test_namespace_lock_fence = test_namespace_lock_fence;
3507 guard
3508 };
3509
3510 Ok(SourceCleanupMutationFence {
3511 guard,
3512 source_lock_covered,
3513 })
3514 }
3515
3516 pub(crate) async fn acquire_all_object_read_locks(
3517 &self,
3518 op: &'static str,
3519 bucket: &str,
3520 object: &str,
3521 opts: &mut ObjectOptions,
3522 ) -> Result<Vec<ObjectLockDiagGuard>> {
3523 let diag_enabled = is_object_lock_diag_enabled();
3524 let mut guards = Vec::with_capacity(self.pools.len() + 1);
3525
3526 let distributed = self.ctx.is_dist_erasure().await;
3530 if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3531 guards.push(guard);
3532 }
3533 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3534 let mut locked_sets = vec![fixed_set];
3535
3536 for pool in &self.pools {
3537 let hashed_set = pool.get_disks_by_key(object);
3538 let mut lock_domain_already_held = !distributed;
3539 if !lock_domain_already_held {
3540 for locked_set in &locked_sets {
3541 if locked_set.shares_namespace_lock_domain(&hashed_set).await {
3542 lock_domain_already_held = true;
3543 break;
3544 }
3545 }
3546 }
3547 if lock_domain_already_held {
3548 continue;
3549 }
3550 let ns_lock = hashed_set.new_ns_lock(bucket, object).await?;
3551 let acquire_start = Instant::now();
3552 let guard = ns_lock
3553 .get_read_lock(get_lock_acquire_timeout())
3554 .await
3555 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3556 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3557 log_object_lock_acquire_if_slow(
3558 op,
3559 bucket,
3560 object,
3561 owner.as_deref(),
3562 ObjectLockDiagMode::Read,
3563 acquire_start.elapsed(),
3564 diag_enabled,
3565 );
3566 guards.push(ObjectLockDiagGuard::new(
3567 guard,
3568 diag_enabled,
3569 op,
3570 diag_enabled.then(|| bucket.to_string()),
3571 diag_enabled.then(|| object.to_string()),
3572 owner,
3573 ObjectLockDiagMode::Read,
3574 ));
3575 locked_sets.push(hashed_set);
3576 }
3577 Ok(guards)
3578 }
3579
3580 pub(crate) async fn acquire_all_physical_object_read_locks(
3586 &self,
3587 op: &'static str,
3588 bucket: &str,
3589 object: &str,
3590 opts: &mut ObjectOptions,
3591 ) -> Result<Vec<ObjectLockDiagGuard>> {
3592 let diag_enabled = is_object_lock_diag_enabled();
3593 let distributed = self.ctx.is_dist_erasure().await;
3594 let mut guards = Vec::new();
3595 if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3596 guards.push(guard);
3597 }
3598
3599 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3600 let mut locked_sets = vec![fixed_set];
3601 for pool in &self.pools {
3602 for set in &pool.disk_set {
3603 let mut lock_domain_already_held = !distributed;
3604 if !lock_domain_already_held {
3605 for locked_set in &locked_sets {
3606 if locked_set.shares_namespace_lock_domain(set).await {
3607 lock_domain_already_held = true;
3608 break;
3609 }
3610 }
3611 }
3612 if lock_domain_already_held {
3613 continue;
3614 }
3615 let ns_lock = set.new_ns_lock(bucket, object).await?;
3616 let acquire_start = Instant::now();
3617 let guard = ns_lock
3618 .get_read_lock(get_lock_acquire_timeout())
3619 .await
3620 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3621 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3622 log_object_lock_acquire_if_slow(
3623 op,
3624 bucket,
3625 object,
3626 owner.as_deref(),
3627 ObjectLockDiagMode::Read,
3628 acquire_start.elapsed(),
3629 diag_enabled,
3630 );
3631 guards.push(ObjectLockDiagGuard::new(
3632 guard,
3633 diag_enabled,
3634 op,
3635 diag_enabled.then(|| bucket.to_string()),
3636 diag_enabled.then(|| object.to_string()),
3637 owner,
3638 ObjectLockDiagMode::Read,
3639 ));
3640 locked_sets.push(Arc::clone(set));
3641 }
3642 }
3643 Ok(guards)
3644 }
3645
3646 pub(crate) async fn acquire_all_physical_object_write_locks(
3647 &self,
3648 op: &'static str,
3649 bucket: &str,
3650 object: &str,
3651 ) -> Result<Vec<ObjectLockDiagGuard>> {
3652 self.acquire_physical_object_write_locks(op, bucket, object, false).await
3653 }
3654
3655 async fn acquire_remaining_physical_object_write_locks(
3656 &self,
3657 op: &'static str,
3658 bucket: &str,
3659 object: &str,
3660 ) -> Result<Vec<ObjectLockDiagGuard>> {
3661 self.acquire_physical_object_write_locks(op, bucket, object, true).await
3662 }
3663
3664 async fn acquire_physical_object_write_locks(
3665 &self,
3666 op: &'static str,
3667 bucket: &str,
3668 object: &str,
3669 store_lock_already_held: bool,
3670 ) -> Result<Vec<ObjectLockDiagGuard>> {
3671 if self.ctx.lock_manager().is_disabled() {
3677 return Err(Error::other("physical object mutation requires namespace locking"));
3678 }
3679 let diag_enabled = is_object_lock_diag_enabled();
3680 let distributed = self.ctx.is_dist_erasure().await;
3681 let mut guards = Vec::new();
3682 if !store_lock_already_held {
3683 guards.push(self.acquire_object_write_lock(op, bucket, object).await?);
3684 }
3685 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3686 let mut locked_sets = vec![fixed_set];
3687 for pool in &self.pools {
3688 for set in &pool.disk_set {
3689 let mut lock_domain_already_held = !distributed;
3690 if !lock_domain_already_held {
3691 for locked_set in &locked_sets {
3692 if locked_set.shares_namespace_lock_domain(set).await {
3693 lock_domain_already_held = true;
3694 break;
3695 }
3696 }
3697 }
3698 if lock_domain_already_held {
3699 continue;
3700 }
3701 let ns_lock = set.new_ns_lock(bucket, object).await?;
3702 let acquire_start = Instant::now();
3703 let guard = ns_lock
3704 .get_write_lock(get_lock_acquire_timeout())
3705 .await
3706 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3707 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3708 log_object_lock_acquire_if_slow(
3709 op,
3710 bucket,
3711 object,
3712 owner.as_deref(),
3713 ObjectLockDiagMode::Write,
3714 acquire_start.elapsed(),
3715 diag_enabled,
3716 );
3717 guards.push(ObjectLockDiagGuard::new(
3718 guard,
3719 diag_enabled,
3720 op,
3721 diag_enabled.then(|| bucket.to_string()),
3722 diag_enabled.then(|| object.to_string()),
3723 owner,
3724 ObjectLockDiagMode::Write,
3725 ));
3726 locked_sets.push(Arc::clone(set));
3727 }
3728 }
3729 Ok(guards)
3730 }
3731
3732 pub(crate) async fn acquire_data_movement_publication_write_locks(
3733 &self,
3734 bucket: &str,
3735 object: &str,
3736 source_pool_idx: usize,
3737 target_pool_idx: usize,
3738 include_fixed_domain: bool,
3739 ) -> Result<Vec<ObjectLockDiagGuard>> {
3740 if self.ctx.lock_manager().is_disabled() {
3741 return Err(Error::other("data movement publication requires namespace locking"));
3742 }
3743 let distributed = self.ctx.is_dist_erasure().await;
3744 let diag_enabled = is_object_lock_diag_enabled();
3745 let mut pool_indices = [source_pool_idx, target_pool_idx];
3746 pool_indices.sort_unstable();
3747 let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3748 let mut locked_sets = Vec::with_capacity(3);
3749 let mut guards = Vec::with_capacity(3);
3750 #[cfg(test)]
3751 let test_namespace_lock_fence = decommission_mutation_fence_for_test(
3752 bucket,
3753 &decode_dir_object(object),
3754 DecommissionMutationFenceTestPhase::Migration,
3755 );
3756
3757 if include_fixed_domain {
3762 let guard = self
3763 .acquire_object_write_lock("data_movement_publication", bucket, object)
3764 .await?;
3765 #[cfg(test)]
3766 let guard = {
3767 let mut guard = guard;
3768 guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3769 guard
3770 };
3771 guards.push(guard);
3772 locked_sets.push(fixed_set);
3773 }
3774 for pool_idx in pool_indices {
3775 let pool = self
3776 .pools
3777 .get(pool_idx)
3778 .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
3779 let set = pool.get_disks_by_key(object);
3780 let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
3781 if !lock_domain_already_held {
3782 for locked_set in &locked_sets {
3783 if locked_set.shares_namespace_lock_domain(&set).await {
3784 lock_domain_already_held = true;
3785 break;
3786 }
3787 }
3788 }
3789 if lock_domain_already_held {
3790 continue;
3791 }
3792 let ns_lock = set.new_ns_lock(bucket, object).await?;
3793 let acquire_start = Instant::now();
3794 let guard = ns_lock
3795 .get_write_lock(get_lock_acquire_timeout())
3796 .await
3797 .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3798 let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3799 log_object_lock_acquire_if_slow(
3800 "data_movement_publication",
3801 bucket,
3802 object,
3803 owner.as_deref(),
3804 ObjectLockDiagMode::Write,
3805 acquire_start.elapsed(),
3806 diag_enabled,
3807 );
3808 let guard = ObjectLockDiagGuard::new(
3809 guard,
3810 diag_enabled,
3811 "data_movement_publication",
3812 diag_enabled.then(|| bucket.to_string()),
3813 diag_enabled.then(|| object.to_string()),
3814 owner,
3815 ObjectLockDiagMode::Write,
3816 );
3817 #[cfg(test)]
3818 let guard = {
3819 let mut guard = guard;
3820 if guards.is_empty() {
3821 guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3822 }
3823 guard
3824 };
3825 guards.push(guard);
3826 locked_sets.push(set);
3827 }
3828 Ok(guards)
3829 }
3830
3831 fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
3832 if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
3833 return reader;
3834 }
3835
3836 if let Some(guard) = guard {
3837 reader.stream = Box::new(LockGuardedReader {
3838 inner: reader.stream,
3839 guard: Some(guard),
3840 });
3841 }
3842
3843 reader
3844 }
3845
3846 async fn get_latest_accessible_object_info_with_idx(
3847 &self,
3848 bucket: &str,
3849 object: &str,
3850 opts: &ObjectOptions,
3851 ) -> Result<(ObjectInfo, usize)> {
3852 let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?;
3853 resolve_latest_object_access(bucket, object, info, idx, opts)
3854 }
3855
3856 pub(crate) async fn select_data_movement_pool_idx(
3857 &self,
3858 bucket: &str,
3859 object: &str,
3860 size: i64,
3861 opts: &ObjectOptions,
3862 no_lock: bool,
3863 ) -> Result<usize> {
3864 let capacity_owner = DecommissionCapacityOwner::from_options(opts);
3865 match self
3866 .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
3867 .await
3868 {
3869 Ok((pinfo, _)) => {
3870 if let Some(owner) = capacity_owner {
3871 if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
3872 return Ok(pinfo.index);
3873 }
3874 } else {
3875 return Ok(pinfo.index);
3876 }
3877 }
3878 Err(err) => {
3879 if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
3880 return Err(err);
3881 }
3882 }
3883 }
3884 if let Some(owner) = capacity_owner {
3885 let expected_data_bytes = opts
3886 .capacity_expected_data_bytes()
3887 .or_else(|| usize::try_from(size).ok())
3888 .unwrap_or_default();
3889 return self
3890 .select_decommission_capacity_target_pool(owner, expected_data_bytes)
3891 .await;
3892 }
3893
3894 self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
3895 }
3896
3897 async fn find_data_movement_target_info(
3898 &self,
3899 bucket: &str,
3900 object: &str,
3901 target_pool_idx: usize,
3902 opts: &ObjectOptions,
3903 ) -> Result<Option<ObjectInfo>> {
3904 let mut lookup_opts = version_aware_lookup_opts(opts, true);
3905 lookup_opts.include_part_checksums = true;
3906
3907 let Some(pool) = self.pools.get(target_pool_idx) else {
3908 return Err(Error::other(format!(
3909 "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
3910 )));
3911 };
3912
3913 match pool.get_object_info(bucket, object, &lookup_opts).await {
3914 Ok(info) => Ok(Some(info)),
3915 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
3916 Err(err) => Err(err),
3917 }
3918 }
3919
3920 async fn has_equivalent_data_movement_delete_marker(
3921 &self,
3922 bucket: &str,
3923 object: &str,
3924 source: &ObjectInfo,
3925 opts: &ObjectOptions,
3926 target_pool_idx: usize,
3927 ) -> Result<bool> {
3928 let equivalent = resolve_data_movement_delete_marker_resume_result(
3929 self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3930 .await,
3931 source,
3932 opts.src_pool_idx,
3933 target_pool_idx,
3934 )?;
3935 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3936 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3937 .await?;
3938 }
3939 Ok(equivalent)
3940 }
3941
3942 async fn has_equivalent_data_movement_tiered_object(
3943 &self,
3944 bucket: &str,
3945 object: &str,
3946 source: &rustfs_filemeta::FileInfo,
3947 opts: &ObjectOptions,
3948 target_pool_idx: usize,
3949 ) -> Result<bool> {
3950 let equivalent = resolve_data_movement_tiered_resume_result(
3951 self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3952 .await,
3953 source,
3954 opts.src_pool_idx,
3955 target_pool_idx,
3956 )?;
3957 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3958 let expected_data_bytes = usize::try_from(source.size).unwrap_or_default();
3959 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
3960 .await?;
3961 }
3962 Ok(equivalent)
3963 }
3964
3965 async fn has_equivalent_data_movement_tier_free_version(
3966 &self,
3967 bucket: &str,
3968 object: &str,
3969 source: &rustfs_filemeta::FileInfo,
3970 opts: &ObjectOptions,
3971 target_pool_idx: usize,
3972 ) -> Result<bool> {
3973 let pool = self
3974 .pools
3975 .get(target_pool_idx)
3976 .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
3977 let equivalent = pool
3978 .get_disks_by_key(object)
3979 .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
3980 .await?;
3981 if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3982 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3983 .await?;
3984 }
3985 Ok(equivalent)
3986 }
3987
3988 fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
3989 result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
3990 }
3991
3992 fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
3993 result.map_err(|err| {
3994 crate::data_movement::data_movement_context_error(
3995 format!("failed to decommission tiered object for {bucket}/{object}: {err}"),
3996 err,
3997 )
3998 })
3999 }
4000
4001 #[instrument(skip(self, fi, opts))]
4002 pub(crate) async fn decommission_tiered_object(
4003 self: &Arc<Self>,
4004 bucket: &str,
4005 object: &str,
4006 fi: &rustfs_filemeta::FileInfo,
4007 opts: &ObjectOptions,
4008 ) -> Result<()> {
4009 check_put_object_args(bucket, object)?;
4010
4011 let mut opts = opts.clone();
4012 let is_free_version = fi.tier_free_version();
4013 if is_free_version {
4014 opts.incl_free_versions = true;
4015 }
4016 let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
4017 None
4018 } else {
4019 let expected = opts
4020 .expected_bucket_incarnation_id
4021 .ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
4022 let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
4023 if let Some(namespace_guard) = guard.namespace_lock_guard() {
4024 opts.add_bucket_lifecycle_lock_guard(namespace_guard);
4025 }
4026 Some(guard)
4027 };
4028
4029 let publication_fence = self
4032 .acquire_tiered_remote_tuple_publication_fence(bucket, object, opts.src_pool_idx, fi)
4033 .await?;
4034
4035 let logical_object = object;
4036 let object = encode_dir_object(logical_object);
4037 ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4038 if self.single_pool() {
4039 return Self::resolve_decommission_tiered_object_result(
4040 Err(Error::other("single pool deployments cannot decommission tiered objects")),
4041 bucket,
4042 &object,
4043 );
4044 }
4045
4046 let idx = if uses_data_movement_pool_selection(&opts) {
4047 Self::resolve_decommission_target_pool_idx_result(
4048 self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
4049 .await,
4050 bucket,
4051 &object,
4052 )?
4053 } else {
4054 Self::resolve_decommission_target_pool_idx_result(
4055 self.get_pool_idx_no_lock(bucket, &object, fi.size).await,
4056 bucket,
4057 &object,
4058 )?
4059 };
4060 #[cfg(all(test, feature = "test-util"))]
4061 if is_free_version {
4062 pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
4063 }
4064 let publication_guard = publication_fence.into_commit_guard(idx, bucket, &object).await?;
4065 publication_guard.add_namespace_lock_fence(&mut opts);
4066 opts.no_lock = true;
4067 let mut fi = publication_guard
4068 .revalidated_tiered_source()
4069 .ok_or_else(|| Error::other("tiered data movement publication guard is missing its revalidated source"))?
4070 .clone();
4071 if opts.data_movement {
4072 crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
4073 }
4074 if opts.data_movement && idx == opts.src_pool_idx {
4075 let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4076 Some(
4077 self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default())
4078 .await?,
4079 )
4080 } else {
4081 self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
4082 .await
4083 };
4084 let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
4085 if is_free_version && target_pool_idx == opts.src_pool_idx {
4086 return Err(Error::DiskFull);
4087 }
4088 let equivalent = if is_free_version {
4089 self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
4090 .await?
4091 } else {
4092 self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
4093 .await?
4094 };
4095 if equivalent {
4096 return Ok(());
4097 }
4098
4099 return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
4100 }
4101
4102 let expected_data_bytes = usize::try_from(fi.size).ok();
4103 let result = self
4104 .run_decommission_capacity_admitted_mutation(
4105 idx,
4106 DecommissionCapacityOwner::from_options(&opts),
4107 expected_data_bytes,
4108 || async {
4109 if is_free_version {
4110 self.pools[idx]
4111 .get_disks_by_key(&object)
4112 .decommission_tier_free_version(bucket, &object, &fi, &opts)
4113 .await
4114 } else {
4115 self.pools[idx]
4116 .get_disks_by_key(&object)
4117 .decommission_tiered_object(bucket, &object, &fi, &opts)
4118 .await
4119 }
4120 },
4121 )
4122 .await;
4123 if matches!(result, Err(Error::PreconditionFailed)) {
4124 if self
4125 .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
4126 .await?
4127 {
4128 return Ok(());
4129 }
4130 return Err(StorageError::DataMovementOverwriteErr(
4131 bucket.to_owned(),
4132 object,
4133 opts.version_id.clone().unwrap_or_default(),
4134 ));
4135 }
4136 if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
4137 return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
4138 }
4139 Self::resolve_decommission_tiered_object_result(result, bucket, &object)
4140 }
4141
4142 pub async fn get_object_reader_for_copy(
4150 &self,
4151 bucket: &str,
4152 object: &str,
4153 range: Option<HTTPRangeSpec>,
4154 h: HeaderMap,
4155 opts: &ObjectOptions,
4156 ) -> Result<(GetObjectReader, tokio_util::sync::CancellationToken)> {
4157 let cancellation = tokio_util::sync::CancellationToken::new();
4158 let reader = crate::set_disk::with_get_object_read_cancellation(
4159 cancellation.clone(),
4160 crate::set_disk::with_get_object_read_policy(
4161 crate::set_disk::GetObjectReadPolicy::CopySource,
4162 self.handle_get_object_reader(bucket, object, range, h, opts),
4163 ),
4164 )
4165 .await?;
4166 Ok((reader, cancellation))
4167 }
4168
4169 #[instrument(level = "debug", skip(self, h))]
4170 #[hotpath::measure(impl_type = "ECStore")]
4171 pub(super) async fn handle_get_object_reader(
4172 &self,
4173 bucket: &str,
4174 object: &str,
4175 range: Option<HTTPRangeSpec>,
4176 h: HeaderMap,
4177 opts: &ObjectOptions,
4178 ) -> Result<GetObjectReader> {
4179 check_get_obj_args(bucket, object)?;
4180
4181 let object = rustfs_utils::path::encode_dir_object_ref(object);
4182 let mut opts = opts.clone();
4183 let read_lock_guard = self
4184 .acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
4185 .await?;
4186
4187 let reader = if self.single_pool() {
4188 self.pools[0]
4189 .get_object_reader(bucket, object.as_ref(), range, h, &opts)
4190 .await?
4191 } else {
4192 Box::pin(async {
4195 let (metadata, idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
4196 if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
4197 return Err(error);
4198 }
4199 self.pools[idx]
4200 .get_object_reader_with_prepared_metadata(bucket, object.as_ref(), range, h, &opts, metadata)
4201 .await
4202 })
4203 .await?
4204 };
4205
4206 Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
4207 }
4208
4209 async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
4210 check_put_object_args(bucket, object)?;
4211
4212 let object = encode_dir_object(object);
4213 let mut opts = opts.clone();
4214 if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
4215 opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
4216 }
4217 if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
4218 let expected_incarnation_id = opts
4219 .expected_bucket_incarnation_id
4220 .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
4221 if opts.object_lock_config_snapshot.is_none() {
4222 opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
4223 }
4224 let snapshot = match opts.object_lock_config_snapshot.as_ref() {
4225 Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
4226 Arc::clone(snapshot)
4227 }
4228 _ => {
4229 return Err(Error::other(
4230 "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
4231 ));
4232 }
4233 };
4234 snapshot.add_lock_fences(&mut opts);
4235 }
4236 Ok((object, opts))
4237 }
4238
4239 async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
4240 if self.single_pool() {
4241 return Ok(0);
4242 }
4243
4244 let idx = if uses_data_movement_pool_selection(opts) {
4245 self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
4246 } else if opts.no_lock {
4247 self.get_pool_idx_no_lock(bucket, object, size).await?
4248 } else {
4249 self.get_pool_idx(bucket, object, size).await?
4250 };
4251
4252 if opts.data_movement && idx == opts.src_pool_idx {
4253 return Err(StorageError::DataMovementOverwriteErr(
4254 bucket.to_owned(),
4255 object.to_owned(),
4256 opts.version_id.clone().unwrap_or_default(),
4257 ));
4258 }
4259 Ok(idx)
4260 }
4261
4262 #[cfg(test)]
4263 pub(crate) async fn put_object_for_data_movement(
4264 self: &Arc<Self>,
4265 bucket: &str,
4266 object: &str,
4267 data: &mut PutObjReader,
4268 opts: &ObjectOptions,
4269 mutation_fence: Option<&ObjectLockDiagGuard>,
4270 ) -> Result<(usize, Result<ObjectInfo>)> {
4271 self.put_object_for_data_movement_inner(bucket, object, data, opts, mutation_fence, None)
4272 .await
4273 }
4274
4275 pub(crate) async fn put_object_for_data_movement_with_publication_fence(
4276 self: &Arc<Self>,
4277 bucket: &str,
4278 object: &str,
4279 data: &mut PutObjReader,
4280 opts: &ObjectOptions,
4281 publication_fence: RemoteTuplePublicationFence,
4282 ) -> Result<(usize, Result<ObjectInfo>)> {
4283 self.put_object_for_data_movement_inner(bucket, object, data, opts, None, Some(publication_fence))
4284 .await
4285 }
4286
4287 async fn put_object_for_data_movement_inner(
4288 self: &Arc<Self>,
4289 bucket: &str,
4290 object: &str,
4291 data: &mut PutObjReader,
4292 opts: &ObjectOptions,
4293 mutation_fence: Option<&ObjectLockDiagGuard>,
4294 publication_fence: Option<RemoteTuplePublicationFence>,
4295 ) -> Result<(usize, Result<ObjectInfo>)> {
4296 if !opts.data_movement {
4297 return Err(Error::other("data movement PUT requires data_movement options"));
4298 }
4299 let request_opts = opts;
4300 let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
4301 ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4302 let idx = self
4303 .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4304 .await?;
4305 let fixed_read_anchor = publication_fence
4306 .as_ref()
4307 .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
4308 self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
4309 .await;
4310 let expected_data_bytes = usize::try_from(data.size()).ok();
4311 let result = self
4312 .run_decommission_capacity_admitted_mutation_with_capacity_lease(
4313 idx,
4314 DecommissionCapacityOwner::from_options(&opts),
4315 expected_data_bytes,
4316 |capacity_lease| async move {
4317 if let Some(capacity_lease) = capacity_lease {
4318 opts.add_namespace_lock_lost_signal(capacity_lease);
4319 }
4320 let result = match publication_fence {
4321 Some(publication_fence) => {
4322 self.pools[idx]
4323 .put_object_with_old_current_size_for_data_movement(
4324 bucket,
4325 &object,
4326 data,
4327 &opts,
4328 publication_fence,
4329 )
4330 .await
4331 }
4332 None => {
4333 self.pools[idx]
4334 .put_object_with_old_current_size(bucket, &object, data, &opts)
4335 .await
4336 }
4337 };
4338 result.map(|(object_info, _)| object_info)
4339 },
4340 )
4341 .await;
4342 let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
4343 if result.is_ok() {
4344 list_objects::observe_list_objects_mutation(self, bucket).await;
4345 }
4346 Ok((idx, result))
4347 }
4348
4349 #[instrument(level = "debug", skip(self, data))]
4350 #[hotpath::measure(impl_type = "ECStore")]
4351 pub(super) async fn handle_put_object(
4352 &self,
4353 bucket: &str,
4354 object: &str,
4355 data: &mut PutObjReader,
4356 opts: &ObjectOptions,
4357 ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
4358 let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
4359 let idx = self
4360 .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4361 .await?;
4362 let mut opts = opts;
4363 if !self.single_pool() {
4364 opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4365 }
4366 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4367 let result = self.pools[idx]
4368 .put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
4369 .await;
4370 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4371 result
4372 }
4373
4374 #[instrument(level = "trace", skip(self))]
4375 pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4376 self.get_object_info_snapshot(bucket, object, opts, false).await
4377 }
4378
4379 #[instrument(level = "trace", skip_all)]
4385 pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4386 self.get_object_info_snapshot(bucket, object, opts, true).await
4387 }
4388
4389 async fn get_object_info_snapshot(
4390 &self,
4391 bucket: &str,
4392 object: &str,
4393 opts: &ObjectOptions,
4394 allow_delete_marker: bool,
4395 ) -> Result<ObjectInfo> {
4396 check_object_args(bucket, object)?;
4397
4398 let object = encode_dir_object(object);
4399 let mut opts = opts.clone();
4400 let _object_lock_guard = self
4401 .acquire_object_read_lock_if_needed("get_object_info", bucket, &object, &mut opts)
4402 .await?;
4403
4404 let info = if self.single_pool() {
4405 self.pools[0].get_object_info(bucket, object.as_str(), &opts).await?
4406 } else if allow_delete_marker {
4407 self.get_latest_object_info_with_idx(bucket, object.as_str(), &opts).await?.0
4408 } else {
4409 self.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
4410 .await?
4411 .0
4412 };
4413 opts.precondition_check(&info)?;
4414 Ok(info)
4415 }
4416
4417 #[instrument(skip(self))]
4418 #[allow(clippy::too_many_arguments)]
4419 pub(super) async fn handle_copy_object(
4420 &self,
4421 src_bucket: &str,
4422 src_object: &str,
4423 dst_bucket: &str,
4424 dst_object: &str,
4425 src_info: &mut ObjectInfo,
4426 src_opts: &ObjectOptions,
4427 dst_opts: &ObjectOptions,
4428 ) -> Result<ObjectInfo> {
4429 check_copy_obj_args(src_bucket, src_object)?;
4430 check_copy_obj_args(dst_bucket, dst_object)?;
4431
4432 let src_object = encode_dir_object(src_object);
4433 let dst_object = encode_dir_object(dst_object);
4434
4435 let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
4436
4437 let mut dst_opts = dst_opts.clone();
4438 dst_opts.inherit_shard_integrity(src_info);
4439 if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
4440 dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
4441 }
4442 let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
4443 None
4444 } else {
4445 Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
4446 };
4447 let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4448 dst_opts.add_bucket_lifecycle_lock_guard(guard);
4449 let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
4450 if dst_opts
4451 .expected_bucket_incarnation_id
4452 .is_some_and(|expected| expected != current_incarnation_id)
4453 {
4454 return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
4455 }
4456 Some(current_incarnation_id)
4457 } else {
4458 dst_opts.expected_bucket_incarnation_id
4459 };
4460 if dst_opts
4461 .bucket_lifecycle_lock_fence
4462 .as_ref()
4463 .is_some_and(NamespaceLockFence::is_lock_lost)
4464 {
4465 return Err(StorageError::NamespaceLockQuorumUnavailable {
4466 mode: "copy_object_bucket_generation",
4467 bucket: dst_bucket.to_string(),
4468 object: dst_object.clone(),
4469 required: 1,
4470 achieved: 0,
4471 });
4472 }
4473 if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) {
4474 let incarnation_id =
4475 current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?;
4476 let lifecycle_fence = dst_opts
4477 .bucket_lifecycle_lock_fence
4478 .as_ref()
4479 .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?;
4480 let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() {
4481 Some(snapshot) => Arc::clone(snapshot),
4482 None => {
4483 self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence)
4484 .await?
4485 }
4486 };
4487 if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) {
4488 return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation"));
4489 }
4490 snapshot.add_lock_fences(&mut dst_opts);
4491 dst_opts.object_lock_config_snapshot = Some(snapshot);
4492 }
4493 let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() {
4494 self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts)
4495 .await?
4496 } else {
4497 None
4498 };
4499
4500 if cp_src_dst_same {
4501 let (_, pool_idx) = self
4502 .get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
4503 .await?;
4504 let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
4505
4506 if source_pool_writable
4507 && let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
4508 && src_vid == dst_vid
4509 {
4510 let capacity_object = dst_object.clone();
4511 return self
4512 .run_external_decommission_capacity_object_mutation(
4513 pool_idx,
4514 dst_bucket,
4515 &capacity_object,
4516 &capacity_object,
4517 dst_opts.clone(),
4518 |opts| async move {
4519 self.pools[pool_idx]
4520 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4521 .await
4522 },
4523 )
4524 .await;
4525 }
4526
4527 if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
4528 if src_info.metadata_only {
4529 let capacity_object = dst_object.clone();
4539 return self
4540 .run_external_decommission_capacity_object_mutation(
4541 pool_idx,
4542 dst_bucket,
4543 &capacity_object,
4544 &capacity_object,
4545 dst_opts.clone(),
4546 |opts| async move {
4547 self.pools[pool_idx]
4548 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4549 .await
4550 },
4551 )
4552 .await;
4553 }
4554 let mut put_opts = ObjectOptions {
4556 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4557 versioned: dst_opts.versioned,
4558 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4559 version_id: dst_opts.version_id.clone(),
4560 no_lock: dst_opts.no_lock,
4561 mod_time: dst_opts.mod_time,
4562 http_preconditions: dst_opts.http_preconditions.clone(),
4563 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4564 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4565 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4566 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4567 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4568 quota_admission: dst_opts.quota_admission,
4569 ..Default::default()
4570 };
4571 if !self.single_pool() {
4572 put_opts.decommission_capacity_admission =
4573 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4574 }
4575 return if let Some(reader) = src_info.put_object_reader.as_mut() {
4576 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4577 let result = self.pools[pool_idx]
4578 .put_object(dst_bucket, &dst_object, reader, &put_opts)
4579 .await;
4580 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4581 result
4582 } else {
4583 Err(StorageError::InvalidArgument(
4584 src_bucket.to_owned(),
4585 src_object.to_owned(),
4586 "put_object_reader is none".to_owned(),
4587 ))
4588 };
4589 }
4590
4591 if source_pool_writable && dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
4592 if let Some(reader) = src_info.put_object_reader.as_mut() {
4598 let mut put_opts = ObjectOptions {
4599 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4600 versioned: dst_opts.versioned,
4601 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4602 version_id: dst_opts.version_id.clone(),
4603 no_lock: dst_opts.no_lock,
4604 mod_time: dst_opts.mod_time,
4605 http_preconditions: dst_opts.http_preconditions.clone(),
4606 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4607 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4608 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4609 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4610 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4611 quota_admission: dst_opts.quota_admission,
4612 ..Default::default()
4613 };
4614 if !self.single_pool() {
4615 put_opts.decommission_capacity_admission =
4616 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4617 }
4618 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4619 let result = self.pools[pool_idx]
4620 .put_object(dst_bucket, &dst_object, reader, &put_opts)
4621 .await;
4622 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4623 return result;
4624 }
4625 src_info.version_only = true;
4626 let capacity_object = dst_object.clone();
4627 return self
4628 .run_external_decommission_capacity_object_mutation(
4629 pool_idx,
4630 dst_bucket,
4631 &capacity_object,
4632 &capacity_object,
4633 dst_opts.clone(),
4634 |opts| async move {
4635 self.pools[pool_idx]
4636 .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4637 .await
4638 },
4639 )
4640 .await;
4641 }
4642 }
4643
4644 let pool_idx = if dst_opts.no_lock {
4645 self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
4646 } else {
4647 self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
4648 };
4649 let dst_object_name = dst_object.as_str();
4650
4651 let mut put_opts = ObjectOptions {
4652 user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4653 versioned: dst_opts.versioned,
4654 shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4655 version_id: dst_opts.version_id.clone(),
4656 no_lock: dst_opts.no_lock,
4657 mod_time: dst_opts.mod_time,
4658 http_preconditions: dst_opts.http_preconditions.clone(),
4659 expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4660 expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4661 namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4662 bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4663 object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4664 quota_admission: dst_opts.quota_admission,
4665 ..Default::default()
4666 };
4667 if !self.single_pool() {
4668 put_opts.decommission_capacity_admission =
4669 crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4670 }
4671
4672 if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
4673 let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4674 let result = self.pools[pool_idx]
4675 .put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
4676 .await;
4677 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4678 return result;
4679 }
4680
4681 Err(StorageError::InvalidArgument(
4682 src_bucket.to_owned(),
4683 src_object.to_owned(),
4684 "put_object_reader is none".to_owned(),
4685 ))
4686 }
4687
4688 pub(super) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> bool {
4695 let prefix = decode_dir_object(object);
4696 for pool in self.pools.iter() {
4697 for set in pool.disk_set.iter() {
4698 if !set.orphan_purge_has_complete_disk_set().await {
4699 return false;
4700 }
4701 }
4702 }
4703
4704 let mut purged = false;
4705 for pool in self.pools.iter() {
4706 for set in pool.disk_set.iter() {
4707 match set.purge_orphan_dir_object(bucket, &prefix).await {
4708 Ok(set_purged) => purged |= set_purged,
4709 Err(err) => {
4710 warn!(
4711 bucket,
4712 prefix,
4713 pool_index = pool.pool_idx,
4714 error = ?err,
4715 "failed to purge orphan directory prefix"
4716 );
4717 }
4718 }
4719 }
4720 }
4721 purged
4722 }
4723
4724 pub(super) async fn purge_orphan_dir_objects_in_bucket(&self, bucket: &str) -> bool {
4725 if is_meta_bucketname(bucket) {
4726 return false;
4727 }
4728 for pool in self.pools.iter() {
4729 for set in pool.disk_set.iter() {
4730 if !set.orphan_purge_has_complete_disk_set().await {
4731 return false;
4732 }
4733 }
4734 }
4735
4736 let mut purged = false;
4737 for pool in self.pools.iter() {
4738 for set in pool.disk_set.iter() {
4739 purged |= set.purge_orphan_dir_objects_in_bucket(bucket).await;
4740 }
4741 }
4742 purged
4743 }
4744
4745 pub async fn delete_object_with_tier_delete_journal(
4746 self: &Arc<Self>,
4747 bucket: &str,
4748 object: &str,
4749 opts: ObjectOptions,
4750 ) -> Result<ObjectInfo> {
4751 let result = self
4752 .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
4753 .await;
4754 if result.is_ok() {
4755 list_objects::observe_list_objects_mutation(self, bucket).await;
4756 }
4757 result
4758 }
4759
4760 pub async fn delete_objects_with_tier_delete_journal(
4761 self: &Arc<Self>,
4762 bucket: &str,
4763 objects: Vec<ObjectToDelete>,
4764 opts: ObjectOptions,
4765 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
4766 let result = self
4767 .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
4768 .await;
4769 let success_count = result.1.iter().filter(|err| err.is_none()).count();
4770 if success_count > 0 {
4771 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4772 }
4773 result
4774 }
4775
4776 pub async fn delete_objects_with_tier_delete_journal_and_accounting(
4777 self: &Arc<Self>,
4778 bucket: &str,
4779 objects: Vec<ObjectToDelete>,
4780 opts: ObjectOptions,
4781 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
4782 let result = self
4783 .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
4784 .await;
4785 let success_count = result.1.iter().filter(|err| err.is_none()).count();
4786 if success_count > 0 {
4787 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4788 }
4789 result
4790 }
4791
4792 #[instrument(skip(self))]
4793 pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
4794 self.handle_delete_object_with_journal(bucket, object, opts, None).await
4795 }
4796
4797 pub(super) async fn handle_delete_object_with_journal(
4798 &self,
4799 bucket: &str,
4800 object: &str,
4801 opts: ObjectOptions,
4802 tier_journal_api: Option<Arc<ECStore>>,
4803 ) -> Result<ObjectInfo> {
4804 Box::pin(async move {
4805 let mut opts = opts;
4806 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4807 let result = self
4808 .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)
4809 .await;
4810 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4811 result
4812 })
4813 .await
4814 }
4815
4816 async fn handle_delete_object_with_journal_inner(
4817 &self,
4818 bucket: &str,
4819 object: &str,
4820 opts: ObjectOptions,
4821 tier_journal_api: Option<Arc<ECStore>>,
4822 ) -> Result<ObjectInfo> {
4823 check_del_obj_args(bucket, object)?;
4824
4825 if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
4826 return Err(Error::other("lifecycle delete-all requires namespace locking"));
4827 }
4828
4829 let _bucket_lifecycle_guard =
4830 if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
4831 None
4832 } else if opts.delete_prefix {
4833 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
4834 } else {
4835 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
4836 };
4837 let object = if opts.delete_prefix && !opts.delete_prefix_object {
4838 object.to_owned()
4839 } else {
4840 encode_dir_object(object)
4841 };
4842 let object = object.as_str();
4843 let mut opts = opts;
4844 ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
4845 let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
4846 Some(get_expiry_configs(self, bucket).await?)
4847 } else {
4848 None
4849 };
4850 opts.tier_delete_journal_api = tier_journal_api.clone();
4851 if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4852 opts.add_bucket_lifecycle_lock_guard(guard);
4853 }
4854
4855 if !is_meta_bucketname(bucket) {
4856 get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
4857 }
4858 let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
4859 Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
4860 } else {
4861 None
4862 };
4863 if let Some(guard) = _object_lock_metadata_guard.as_ref() {
4864 opts.add_namespace_lock_guard(guard);
4865 }
4866 let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
4867 let (state, incarnation_id, config_revision) =
4868 get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
4869 opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
4870 self.id,
4871 bucket,
4872 incarnation_id,
4873 config_revision,
4874 state,
4875 opts.bucket_lifecycle_lock_fence
4876 .clone()
4877 .unwrap_or_else(NamespaceLockFence::new),
4878 Arc::clone(metadata_guard),
4879 )));
4880 Some(incarnation_id)
4881 } else {
4882 None
4883 };
4884 if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
4885 && expected != current
4886 {
4887 return Err(StorageError::BucketNotFound(bucket.to_string()));
4888 }
4889 if opts.expected_bucket_incarnation_id.is_none() {
4890 opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
4891 }
4892 #[cfg(any(test, feature = "test-util"))]
4893 pause_delete_after_object_lock_snapshot(bucket).await;
4894
4895 if opts.delete_prefix && !opts.delete_prefix_object {
4896 delete_recursive_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4897 return Ok(ObjectInfo::default());
4898 }
4899
4900 let _object_lock_guard = if opts.expected_current_version_id.is_none() {
4901 self.acquire_object_write_lock_if_needed("delete_object", bucket, object, &mut opts)
4902 .await?
4903 } else {
4904 None
4905 };
4906 #[cfg(any(test, feature = "test-util"))]
4907 if _object_lock_guard.is_some() {
4908 notify_delete_namespace_acquired(bucket);
4909 }
4910 if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
4911 let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
4912 let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
4913 if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
4914 return Err(StorageError::PreconditionFailed);
4915 }
4916 let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
4917 let (mut current, _) = self
4918 .get_latest_object_info_with_idx(
4919 bucket,
4920 object,
4921 &ObjectOptions {
4922 no_lock: true,
4923 metadata_cache_safe: false,
4924 ..Default::default()
4925 },
4926 )
4927 .await?;
4928 let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
4929 if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
4930 return Err(StorageError::PreconditionFailed);
4931 }
4932 current.name = decode_dir_object(¤t.name);
4933 let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), ¤t).await;
4934 if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
4935 return Err(StorageError::PreconditionFailed);
4936 }
4937 }
4938 if opts.delete_prefix {
4939 delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4940 return Ok(ObjectInfo::default());
4941 }
4942
4943 let creates_latest_marker = latest_versioned_delete_creates_distinct_marker(&opts)
4944 || (opts.version_suspended && should_create_delete_marker_for_missing_object(&opts));
4945 let mut gopts = delete_pool_lookup_opts(&opts, true);
4946 if creates_latest_marker {
4947 gopts.skip_rebalancing = false;
4950 }
4951
4952 if opts.data_movement {
4953 let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
4954 let existing_pool_idx = existing_pool_info
4955 .as_ref()
4956 .map(|(pinfo, _)| pinfo.index)
4957 .map_err(Clone::clone);
4958 let selected_target_pool_idx =
4959 match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
4960 Some(pool_idx) => pool_idx,
4961 None => {
4962 if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4963 self.select_decommission_capacity_target_pool(owner, 0).await?
4964 } else {
4965 self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4966 .await
4967 .ok_or(Error::DiskFull)?
4968 }
4969 }
4970 };
4971 let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
4972 if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4973 Some(self.select_decommission_capacity_target_pool(owner, 0).await?)
4974 } else {
4975 self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4976 .await
4977 }
4978 } else {
4979 None
4980 };
4981 let target_pool_idx =
4982 resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
4983 let mut delete_marker_target_opts = None;
4984
4985 if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4986 let source = self
4987 .find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
4988 .await?;
4989 let Some(source) = source else {
4990 return Err(StorageError::DataMovementOverwriteErr(
4991 bucket.to_owned(),
4992 object.to_owned(),
4993 opts.version_id.unwrap_or_default(),
4994 ));
4995 };
4996 if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
4997 return Err(StorageError::DataMovementOverwriteErr(
4998 bucket.to_owned(),
4999 object.to_owned(),
5000 opts.version_id.unwrap_or_default(),
5001 ));
5002 }
5003 let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
5004 return Err(StorageError::DataMovementOverwriteErr(
5005 bucket.to_owned(),
5006 object.to_owned(),
5007 opts.version_id.unwrap_or_default(),
5008 ));
5009 };
5010 let target = self
5011 .find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
5012 .await?;
5013 if let Some(target) = target {
5014 if is_equivalent_data_movement_delete_marker(&source, &target) {
5015 if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) {
5016 self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
5017 .await?;
5018 }
5019 let mut target = target;
5020 target.name = decode_dir_object(object);
5021 return Ok(target);
5022 }
5023 return Err(StorageError::DataMovementOverwriteErr(
5024 bucket.to_owned(),
5025 object.to_owned(),
5026 opts.version_id.unwrap_or_default(),
5027 ));
5028 }
5029 delete_marker_target_opts = Some(target_opts);
5030 }
5031
5032 if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
5033 if let Ok((source_pool_info, _)) = existing_pool_info
5034 && opts.delete_marker
5035 && is_data_movement_delete_marker(&source_pool_info.object_info)
5036 && self
5037 .has_equivalent_data_movement_delete_marker(
5038 bucket,
5039 object,
5040 &source_pool_info.object_info,
5041 &opts,
5042 target_pool_idx,
5043 )
5044 .await?
5045 {
5046 let mut obj = source_pool_info.object_info;
5047 obj.name = decode_dir_object(object);
5048 return Ok(obj);
5049 }
5050
5051 return Err(StorageError::DataMovementOverwriteErr(
5052 bucket.to_owned(),
5053 object.to_owned(),
5054 opts.version_id.unwrap_or_default(),
5055 ));
5056 }
5057
5058 let target_opts = delete_marker_target_opts.unwrap_or(opts);
5059 let mut obj = self
5060 .run_decommission_capacity_admitted_mutation_with_capacity_lease(
5061 target_pool_idx,
5062 DecommissionCapacityOwner::from_options(&target_opts),
5063 None,
5064 |capacity_lease| async move {
5065 let mut target_opts = target_opts;
5066 if let Some(capacity_lease) = capacity_lease {
5067 target_opts.add_namespace_lock_lost_signal(capacity_lease);
5068 }
5069 self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await
5070 },
5071 )
5072 .await?;
5073 obj.name = decode_dir_object(obj.name.as_str());
5074 return Ok(obj);
5075 }
5076
5077 let existing_pool_info = if creates_latest_marker {
5079 self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
5080 } else {
5081 self.get_pool_info_existing_with_opts(bucket, object, &gopts).await
5082 };
5083 let (mut pinfo, errs) = match existing_pool_info {
5084 Ok(res) => res,
5085 Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
5086 Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
5087 let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5088 let pool = self.pools[target_pool_idx].clone();
5089 let mut obj = self
5090 .run_external_decommission_capacity_object_mutation(
5091 target_pool_idx,
5092 bucket,
5093 object,
5094 object,
5095 opts,
5096 |opts| async move { pool.delete_object(bucket, object, opts).await },
5097 )
5098 .await?;
5099 #[cfg(test)]
5100 pause_versioned_delete_marker_after_commit(bucket, object).await;
5101 obj.name = decode_dir_object(object);
5102 return Ok(obj);
5103 }
5104 Err(err) => {
5105 if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
5110 return Ok(ObjectInfo {
5111 bucket: bucket.to_owned(),
5112 name: decode_dir_object(object),
5113 ..Default::default()
5114 });
5115 }
5116 return Err(err);
5117 }
5118 };
5119
5120 if creates_latest_marker && self.is_suspended(pinfo.index).await {
5121 let has_active_reservation = self
5122 .pool_meta
5123 .read()
5124 .await
5125 .has_active_decommission_capacity_reservation(pinfo.index);
5126 if has_active_reservation {
5127 pinfo.index = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5128 }
5129 }
5130
5131 if pinfo.object_info.delete_marker && opts.version_id.is_none() && !creates_latest_marker {
5132 pinfo.object_info.name = decode_dir_object(object);
5133 return Ok(pinfo.object_info);
5134 }
5135
5136 if opts.data_movement && opts.src_pool_idx == pinfo.index {
5137 return Err(StorageError::DataMovementOverwriteErr(
5138 bucket.to_owned(),
5139 object.to_owned(),
5140 opts.version_id.unwrap_or_default(),
5141 ));
5142 }
5143
5144 if should_delete_from_all_pools(&opts, errs.len()) {
5145 let mut obj = self
5146 .delete_object_from_all_pools(bucket, object, &opts, &pinfo.object_info, errs)
5147 .await?;
5148 obj.name = decode_dir_object(object);
5149 return Ok(obj);
5150 }
5151
5152 for pool in self.pools.iter() {
5153 if creates_latest_marker && pool.pool_idx != pinfo.index {
5154 continue;
5155 }
5156 if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
5157 if creates_latest_marker {
5158 return Err(StorageError::SlowDown);
5159 }
5160 continue;
5161 }
5162
5163 let pool_idx = pool.pool_idx;
5164 let pool = pool.clone();
5165 match self
5166 .run_external_decommission_capacity_object_delete(
5167 pool_idx,
5168 bucket,
5169 object,
5170 object,
5171 opts.clone(),
5172 |opts| async move { pool.delete_object(bucket, object, opts).await },
5173 )
5174 .await
5175 {
5176 Ok(res) => {
5177 #[cfg(test)]
5178 pause_versioned_delete_marker_after_commit(bucket, object).await;
5179 let mut obj = res;
5180 obj.name = decode_dir_object(object);
5181 return Ok(obj);
5182 }
5183 Err(err) => {
5184 if creates_latest_marker || (!is_err_object_not_found(&err) && !is_err_version_not_found(&err)) {
5185 return Err(err);
5186 }
5187 }
5188 }
5189 }
5190
5191 if let Some(ver) = opts.version_id {
5192 return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
5193 }
5194
5195 Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
5196 }
5197
5198 #[instrument(skip(self, objects, opts))]
5199 pub(super) async fn handle_delete_objects(
5200 &self,
5201 bucket: &str,
5202 objects: Vec<ObjectToDelete>,
5203 opts: ObjectOptions,
5204 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5205 self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
5206 }
5207
5208 pub(super) async fn handle_delete_objects_with_journal(
5209 &self,
5210 bucket: &str,
5211 objects: Vec<ObjectToDelete>,
5212 opts: ObjectOptions,
5213 tier_journal_api: Option<Arc<ECStore>>,
5214 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5215 let (deleted, errors, _) = self
5216 .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
5217 .await;
5218 (deleted, errors)
5219 }
5220
5221 pub(super) async fn handle_delete_objects_with_journal_and_accounting(
5222 &self,
5223 bucket: &str,
5224 objects: Vec<ObjectToDelete>,
5225 opts: ObjectOptions,
5226 tier_journal_api: Option<Arc<ECStore>>,
5227 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5228 Box::pin(async move {
5229 let mut opts = opts;
5230 let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
5231 let result = self
5232 .handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
5233 .await;
5234 enqueue_recorded_tier_free_versions(self, receipt_sink).await;
5235 result
5236 })
5237 .await
5238 }
5239
5240 async fn handle_delete_objects_with_journal_and_accounting_inner(
5241 &self,
5242 bucket: &str,
5243 objects: Vec<ObjectToDelete>,
5244 opts: ObjectOptions,
5245 tier_journal_api: Option<Arc<ECStore>>,
5246 ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5247 let objects: Vec<ObjectToDelete> = objects
5249 .iter()
5250 .map(|v| {
5251 let mut v = v.clone();
5252 v.object_name = encode_dir_object(v.object_name.as_str());
5253 v
5254 })
5255 .collect();
5256
5257 let mut del_objects = vec![DeletedObject::default(); objects.len()];
5259 let mut accounting = vec![None; objects.len()];
5260
5261 let mut del_errs = Vec::with_capacity(objects.len());
5262 for _ in 0..objects.len() {
5263 del_errs.push(None)
5264 }
5265
5266 let mut opts = opts;
5267 opts.tier_delete_journal_api = tier_journal_api;
5268 let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5269 None
5270 } else {
5271 match self.acquire_bucket_lifecycle_read_lock(bucket).await {
5272 Ok(guard) => Some(guard),
5273 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5274 }
5275 };
5276 if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
5277 opts.add_bucket_lifecycle_lock_guard(guard);
5278 }
5279 if opts.delete_replication_config_snapshot.is_none() {
5280 match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
5281 Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
5282 Err(err) => {
5283 let message = err.to_string();
5284 let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
5285 return (del_objects, errors, accounting);
5286 }
5287 }
5288 }
5289 if !is_meta_bucketname(bucket)
5290 && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
5291 {
5292 return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
5293 }
5294 let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
5295 None
5296 } else {
5297 Some(Arc::new(
5298 match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
5299 Ok(guard) => guard,
5300 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5301 },
5302 ))
5303 };
5304 if let Some(guard) = _object_lock_metadata_guard.as_ref() {
5305 opts.add_namespace_lock_guard(guard);
5306 }
5307 let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
5308 let (state, incarnation_id, config_revision) =
5309 match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
5310 Ok(snapshot) => snapshot,
5311 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5312 };
5313 opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
5314 self.id,
5315 bucket,
5316 incarnation_id,
5317 config_revision,
5318 state,
5319 opts.bucket_lifecycle_lock_fence
5320 .clone()
5321 .unwrap_or_else(NamespaceLockFence::new),
5322 Arc::clone(metadata_guard),
5323 )));
5324 Some(incarnation_id)
5325 } else {
5326 None
5327 };
5328 if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
5329 && expected != current
5330 {
5331 return return_batch_delete_lock_error_with_accounting(
5332 objects.as_slice(),
5333 StorageError::BucketNotFound(bucket.to_string()),
5334 );
5335 }
5336 if opts.expected_bucket_incarnation_id.is_none() {
5337 opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
5338 }
5339 #[cfg(any(test, feature = "test-util"))]
5340 if current_bucket_incarnation_id.is_some() {
5341 pause_delete_after_object_lock_snapshot(bucket).await;
5342 }
5343 let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
5344 Ok(guards) => guards,
5345 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5346 };
5347 #[cfg(any(test, feature = "test-util"))]
5348 if !_object_lock_guards.is_empty() {
5349 notify_delete_namespace_acquired(bucket);
5350 }
5351
5352 let delete_config_snapshot = opts
5353 .delete_replication_config_snapshot
5354 .as_deref()
5355 .expect("batch delete replication config snapshot should be loaded");
5356 let latest_marker_objects = objects
5357 .iter()
5358 .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot))
5359 .collect::<Vec<_>>();
5360 let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map(
5361 |(object, creates_marker)| async move {
5362 if *creates_marker {
5363 Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await)
5364 } else {
5365 None
5366 }
5367 },
5368 ))
5369 .await;
5370 let mut marker_target_pool_indices = Vec::with_capacity(objects.len());
5371 for (idx, target_result) in marker_target_results.into_iter().enumerate() {
5372 match target_result {
5373 Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)),
5374 Some(Err(err)) => {
5375 del_errs[idx] = Some(err);
5376 marker_target_pool_indices.push(None);
5377 }
5378 None => marker_target_pool_indices.push(None),
5379 }
5380 }
5381
5382 let _capacity_fence = if !self.single_pool() && latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
5383 let mut target_pool_indices = marker_target_pool_indices.iter().flatten().copied().collect::<Vec<_>>();
5387 target_pool_indices.sort_unstable();
5388 target_pool_indices.dedup();
5389 match self
5390 .acquire_external_decommission_capacity_fence(&target_pool_indices, DecommissionCapacityAdmission::BatchDelete)
5391 .await
5392 {
5393 Ok(fence) => Some(fence),
5394 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5395 }
5396 } else {
5397 None
5398 };
5399
5400 let mut futures = Vec::with_capacity(self.pools.len());
5401 for pool in self.pools.iter() {
5402 if self.is_pool_rebalancing(pool.pool_idx).await {
5403 continue;
5404 }
5405
5406 let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects
5407 .iter()
5408 .enumerate()
5409 .filter(|(idx, _)| {
5410 batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx)
5411 })
5412 .map(|(idx, object)| (idx, object.clone()))
5413 .unzip();
5414 if pool_objects.is_empty() {
5415 continue;
5416 }
5417
5418 let pool_opts = opts.clone();
5419 futures.push(async move {
5420 #[cfg(test)]
5421 let pool_object_names = pool_objects
5422 .iter()
5423 .map(|object| object.object_name.clone())
5424 .collect::<Vec<_>>();
5425 let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
5426 #[cfg(test)]
5427 let result = {
5428 let mut result = result;
5429 let (deleted, errors, _) = &mut result;
5430 inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
5431 result
5432 };
5433 (object_indices, result)
5434 });
5435 }
5436
5437 let results = join_all(futures).await;
5438
5439 for idx in 0..del_objects.len() {
5440 let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
5441 let pool_object_idx = object_indices.binary_search(&idx).ok()?;
5442 Some((&dels[pool_object_idx], &errs[pool_object_idx]))
5443 });
5444 let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results);
5445 if let Some(deleted) = deleted {
5446 del_objects[idx] = deleted;
5447 }
5448 del_errs[idx] = error;
5449
5450 if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] {
5451 del_objects[idx] = DeletedObject {
5452 object_name: objects[idx].object_name.clone(),
5453 version_id: objects[idx].version_id,
5454 ..Default::default()
5455 };
5456 del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone()));
5457 }
5458 }
5459
5460 for (object_indices, (_, _, pool_accounting)) in &results {
5461 for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
5462 accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
5463 }
5464 }
5465
5466 #[cfg(test)]
5467 for (idx, object) in objects.iter().enumerate() {
5468 if del_errs[idx].is_none() && del_objects[idx].delete_marker {
5469 pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await;
5470 }
5471 }
5472
5473 del_objects.iter_mut().for_each(|v| {
5474 v.object_name = decode_dir_object(&v.object_name);
5475 });
5476
5477 (del_objects, del_errs, accounting)
5478
5479 }
5605
5606 #[instrument(skip(self))]
5607 pub(super) async fn handle_add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
5608 let object = encode_dir_object(object);
5609
5610 if self.single_pool() {
5611 let _ = self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
5612 return Ok(());
5613 }
5614
5615 let opts = ObjectOptions {
5616 version_id: Some(version_id.to_string()),
5617 ..Default::default()
5618 };
5619 let (_, idx) = self
5620 .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5621 .await?;
5622
5623 let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
5624 Ok(())
5625 }
5626
5627 #[instrument(skip(self))]
5628 pub(super) async fn handle_transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5629 let object = encode_dir_object(object);
5630 if self.single_pool() {
5631 return self.pools[0].transition_object(bucket, &object, opts).await;
5632 }
5633
5634 let opts = transition_restore_pool_opts(opts);
5635 let (_, idx) = self
5636 .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
5637 .await?;
5638
5639 self.pools[idx].transition_object(bucket, &object, &opts).await
5640 }
5641
5642 #[instrument(skip(self))]
5643 pub(super) async fn handle_restore_transitioned_object(
5644 self: Arc<Self>,
5645 bucket: &str,
5646 object: &str,
5647 opts: &ObjectOptions,
5648 ) -> Result<()> {
5649 let object = encode_dir_object(object);
5650 let mut opts = transition_restore_pool_opts(opts);
5651 if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
5652 opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
5653 }
5654 let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5655 None
5656 } else {
5657 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
5658 };
5659 if let Some(guard) = bucket_lifecycle_guard.as_ref() {
5660 opts.add_bucket_lifecycle_lock_guard(guard);
5661 }
5662 if !is_meta_bucketname(bucket) {
5663 let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
5664 if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
5665 return Err(StorageError::BucketNotFound(bucket.to_string()));
5666 }
5667 }
5668 if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
5669 let expected_incarnation_id = opts
5670 .expected_bucket_incarnation_id
5671 .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?;
5672 let lifecycle_fence = opts
5673 .bucket_lifecycle_lock_fence
5674 .as_ref()
5675 .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?;
5676 let snapshot = match opts.object_lock_config_snapshot.as_ref() {
5677 Some(snapshot) => Arc::clone(snapshot),
5678 None => {
5679 self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
5680 .await?
5681 }
5682 };
5683 if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
5684 return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation"));
5685 }
5686 snapshot.add_lock_fences(&mut opts);
5687 opts.object_lock_config_snapshot = Some(snapshot);
5688 }
5689 let object_name = object.as_str();
5704 if self.single_pool() {
5705 return self.pools[0]
5706 .clone()
5707 .restore_transitioned_object(bucket, object_name, &opts)
5708 .await;
5709 }
5710
5711 let (_, idx) = self
5712 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock))
5713 .await?;
5714
5715 opts.decommission_capacity_admission = Some(Arc::clone(&self));
5716 self.pools[idx]
5717 .clone()
5718 .restore_transitioned_object(bucket, object_name, &opts)
5719 .await
5720 }
5721
5722 #[instrument(skip(self))]
5723 pub(super) async fn handle_put_object_metadata(
5724 &self,
5725 bucket: &str,
5726 object: &str,
5727 opts: &ObjectOptions,
5728 ) -> Result<ObjectInfo> {
5729 let object = encode_dir_object(object);
5730 let mut opts = opts.clone();
5731 opts.metadata_chg = true;
5732 let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5733 None
5734 } else {
5735 let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
5736 let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
5737 if opts
5738 .expected_bucket_incarnation_id
5739 .is_some_and(|expected| expected != current_incarnation_id)
5740 {
5741 return Err(StorageError::BucketNotFound(bucket.to_string()));
5742 }
5743 opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
5744 opts.add_bucket_lifecycle_lock_guard(&guard);
5745 if guard.is_lock_lost() {
5746 return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
5747 }
5748 Some(guard)
5749 };
5750
5751 if self.single_pool() {
5752 let pool = self.pools[0].clone();
5753 let capacity_object = object.clone();
5754 return self
5755 .run_external_decommission_capacity_object_mutation(
5756 0,
5757 bucket,
5758 capacity_object.as_str(),
5759 capacity_object.as_str(),
5760 opts,
5761 |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5762 )
5763 .await;
5764 }
5765
5766 let (_, idx) = self
5767 .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5768 .await?;
5769
5770 let pool = self.pools[idx].clone();
5771 let capacity_object = object.clone();
5772 let result = self
5773 .run_external_decommission_capacity_object_mutation(
5774 idx,
5775 bucket,
5776 capacity_object.as_str(),
5777 capacity_object.as_str(),
5778 opts,
5779 |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5780 )
5781 .await;
5782 drop(bucket_lifecycle_guard);
5783 result
5784 }
5785
5786 #[instrument(skip(self))]
5787 pub(super) async fn handle_get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
5788 let object = encode_dir_object(object);
5789
5790 if self.single_pool() {
5791 return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
5792 }
5793
5794 let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
5795 Ok((*oi.user_tags).clone())
5796 }
5797
5798 #[instrument(level = "debug", skip(self))]
5799 pub(super) async fn handle_put_object_tags(
5800 &self,
5801 bucket: &str,
5802 object: &str,
5803 tags: &str,
5804 opts: &ObjectOptions,
5805 ) -> Result<ObjectInfo> {
5806 let object = encode_dir_object(object);
5807 let object_name = object.as_str();
5808
5809 if self.single_pool() {
5810 return self
5811 .run_external_decommission_capacity_object_mutation(
5812 0,
5813 bucket,
5814 object_name,
5815 object_name,
5816 opts.clone(),
5817 |opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await },
5818 )
5819 .await;
5820 }
5821
5822 let (_, idx) = self
5823 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5824 .await?;
5825
5826 self.run_external_decommission_capacity_object_mutation(
5827 idx,
5828 bucket,
5829 object_name,
5830 object_name,
5831 opts.clone(),
5832 |opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await },
5833 )
5834 .await
5835 }
5836
5837 #[instrument(skip(self))]
5838 pub(super) async fn handle_delete_object_version(
5839 &self,
5840 bucket: &str,
5841 object: &str,
5842 fi: &FileInfo,
5843 force_del_marker: bool,
5844 ) -> Result<()> {
5845 check_del_obj_args(bucket, object)?;
5846
5847 let object = encode_dir_object(object);
5848
5849 if self.single_pool() {
5850 return self.pools[0]
5851 .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
5852 .await;
5853 }
5854 Err(StorageError::NotImplemented)
5855 }
5856
5857 #[instrument(skip(self))]
5858 pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
5859 let object = encode_dir_object(object);
5860 let object_name = object.as_str();
5861
5862 if self.single_pool() {
5863 return self
5864 .run_external_decommission_capacity_object_mutation(
5865 0,
5866 bucket,
5867 object_name,
5868 object_name,
5869 opts.clone(),
5870 |opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await },
5871 )
5872 .await;
5873 }
5874
5875 let (_, idx) = self
5876 .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5877 .await?;
5878
5879 self.run_external_decommission_capacity_object_mutation(
5880 idx,
5881 bucket,
5882 object_name,
5883 object_name,
5884 opts.clone(),
5885 |opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await },
5886 )
5887 .await
5888 }
5889
5890 pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5891 let get_object_reader = <Self as crate::storage_api_contracts::object::ObjectIO>::get_object_reader(
5892 self,
5893 bucket,
5894 object,
5895 None,
5896 HeaderMap::new(),
5897 opts,
5898 )
5899 .await?;
5900 let mut reader = get_object_reader.stream;
5902 tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
5903 Ok(())
5904 }
5905}
5906
5907#[cfg(test)]
5908mod tests {
5909 use super::*;
5910 use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
5911 use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
5912 use crate::bucket::metadata_sys::ObjectLockConfigState;
5913 use crate::bucket::replication::{
5914 ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
5915 version_purge_statuses_map,
5916 };
5917 use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
5918 use crate::core::sets::make_local_two_set_sets_with_ctx;
5919 use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx};
5920 use crate::layout::{
5921 endpoints::{Endpoints, PoolEndpoints, SetupType},
5922 format::FormatV3,
5923 };
5924 use crate::object_api::{
5925 GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
5926 lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
5927 };
5928 use crate::set_disk::same_distributed_lock_domain;
5929 use crate::set_disk::{SetDisks, disk_call_counters};
5930 use crate::storage_api_contracts::bucket::MakeBucketOptions;
5931 use crate::storage_api_contracts::lifecycle::TransitionedObject;
5932 use bytes::Bytes;
5933 use std::future::Future as _;
5934 use std::io::Cursor;
5935 use std::sync::Arc;
5936 use std::sync::atomic::{AtomicUsize, Ordering};
5937 use tokio::io::AsyncReadExt;
5938
5939 #[test]
5940 fn predecessor_replay_scope_distinguishes_exact_object_from_recursive_prefix() {
5941 let source = TierDeleteSourceIdentity {
5942 bucket: "bucket".to_string(),
5943 object: "directory/child".to_string(),
5944 version_id: Some("version".to_string()),
5945 versioned: true,
5946 version_suspended: false,
5947 data_dir: Some("data-dir".to_string()),
5948 etag: Some("etag".to_string()),
5949 mod_time: Some("mod-time".to_string()),
5950 };
5951 let persisted_prefix = encode_dir_object("directory/");
5952
5953 assert!(tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, false,));
5954 assert!(!tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, true,));
5955 let mut exact_source = source;
5956 exact_source.object = "directory/".to_string();
5957 assert!(tier_delete_source_matches_replay_scope(&exact_source, "bucket", &persisted_prefix, true,));
5958 }
5959
5960 struct WaitForLockLossReader {
5961 inner: Cursor<Vec<u8>>,
5962 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5963 resume: ParkingMutex<std::sync::mpsc::Receiver<()>>,
5964 }
5965
5966 impl AsyncRead for WaitForLockLossReader {
5967 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5968 let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
5969 if let Some(poll_started) = self.poll_started.take() {
5970 let _ = poll_started.send(());
5971 if self.resume.lock().recv_timeout(Duration::from_secs(10)).is_err() {
5972 return Poll::Ready(Err(std::io::Error::new(
5973 std::io::ErrorKind::TimedOut,
5974 "lock-loss signal was not observed during read",
5975 )));
5976 }
5977 }
5978 poll
5979 }
5980 }
5981
5982 struct PermanentlyPendingReader {
5983 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5984 }
5985
5986 impl AsyncRead for PermanentlyPendingReader {
5987 fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5988 if let Some(poll_started) = self.poll_started.take() {
5989 let _ = poll_started.send(());
5990 }
5991 Poll::Pending
5992 }
5993 }
5994
5995 struct CommitLateBodyReader {
5996 inner: Cursor<Vec<u8>>,
5997 poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5998 release: Option<tokio::sync::oneshot::Receiver<()>>,
5999 }
6000
6001 impl AsyncRead for CommitLateBodyReader {
6002 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
6003 if let Some(poll_started) = self.poll_started.take() {
6004 let _ = poll_started.send(());
6005 }
6006 if let Some(release) = self.release.as_mut() {
6007 match Pin::new(release).poll(cx) {
6008 Poll::Pending => return Poll::Pending,
6009 Poll::Ready(Ok(())) => self.release = None,
6010 Poll::Ready(Err(_)) => {
6011 return Poll::Ready(Err(std::io::Error::new(
6012 std::io::ErrorKind::BrokenPipe,
6013 "commit-late body release sender was dropped",
6014 )));
6015 }
6016 }
6017 }
6018 Pin::new(&mut self.inner).poll_read(cx, buf)
6019 }
6020 }
6021
6022 struct CountingMissHook {
6023 calls: AtomicUsize,
6024 }
6025
6026 async fn refresh_failure_test_guard(
6027 owner: &'static str,
6028 ) -> (
6029 ObjectLockDiagGuard,
6030 Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6031 Arc<RefreshLossLockClient>,
6032 ) {
6033 let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
6034 rustfs_lock::FastObjectLockManager::new(),
6035 )));
6036 let client = Arc::new(RefreshLossLockClient::with_manager(manager));
6037 let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
6038 owner.to_string(),
6039 vec![Arc::clone(&client) as Arc<dyn rustfs_lock::LockClient>],
6040 1,
6041 );
6042 let request =
6043 rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Shared, owner)
6044 .with_ttl(Duration::from_secs(2))
6045 .with_refresh_interval(Duration::from_millis(20));
6046 let guard = namespace_lock
6047 .acquire_guard(&request)
6048 .await
6049 .expect("distributed read lock acquisition should succeed")
6050 .expect("distributed read lock should reach quorum");
6051 let guard = ObjectLockDiagGuard::new(
6052 guard,
6053 false,
6054 "SelectObjectContent",
6055 Some("bucket".to_string()),
6056 Some("object".to_string()),
6057 Some(owner.to_string()),
6058 ObjectLockDiagMode::Read,
6059 );
6060 let signal = guard
6061 .lock_lost_signal()
6062 .expect("a distributed guard should expose a lock-loss signal");
6063 (guard, signal, client)
6064 }
6065
6066 async fn refresh_failure_test_lease(
6067 owner: &'static str,
6068 ) -> (
6069 Arc<SelectObjectSnapshotLease>,
6070 Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6071 Arc<RefreshLossLockClient>,
6072 ) {
6073 let (guard, signal, client) = refresh_failure_test_guard(owner).await;
6074 (Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client)
6075 }
6076
6077 #[async_trait::async_trait]
6078 impl GetObjectBodyCacheHook for CountingMissHook {
6079 async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
6080 self.calls.fetch_add(1, Ordering::Relaxed);
6081 None
6082 }
6083 }
6084
6085 struct BodyCacheHookGuard;
6086
6087 #[test]
6088 fn select_snapshot_deduplicates_same_distributed_lock_domain() {
6089 let first: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6090 let second: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6091 let other: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6092
6093 assert!(same_distributed_lock_domain(
6094 &[Arc::clone(&first), Arc::clone(&second)],
6095 &[Arc::clone(&second), Arc::clone(&first)]
6096 ));
6097 assert!(same_distributed_lock_domain(
6098 &[Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)],
6099 &[Arc::clone(&second), Arc::clone(&first)]
6100 ));
6101 assert!(!same_distributed_lock_domain(&[first, second], &[other]));
6102 }
6103
6104 #[tokio::test]
6105 async fn publication_commit_locks_dedupe_dist_sets_with_same_clients() {
6106 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6107 let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6108 let mut second_set = (*original_sets.disk_set[1]).clone();
6109 second_set.lockers = original_sets.disk_set[0].lockers.clone();
6110 let mut sets = (*original_sets).clone();
6111 sets.disk_set[1] = Arc::new(second_set);
6112 let sets = Arc::new(sets);
6113 ctx.update_erasure_type(SetupType::DistErasure).await;
6114
6115 assert!(
6116 sets.disk_set[0]
6117 .lockers
6118 .iter()
6119 .zip(&sets.disk_set[1].lockers)
6120 .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)),
6121 "the regression requires identical distributed lock clients"
6122 );
6123 assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index);
6124
6125 let pool_config = sets.endpoints.clone();
6126 let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6127 let object = (0..1_000)
6128 .map(|index| format!("decommission-dist-domain-{index}.bin"))
6129 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6130 .expect("a key should hash to the second set namespace");
6131 let publication_guards = store
6132 .acquire_data_movement_publication_write_locks("bucket", &encode_dir_object(&object), 0, 0, true)
6133 .await
6134 .expect("the publication commit lock set should be acquired");
6135 let target_lock = sets.disk_set[1]
6136 .new_ns_lock("bucket", &object)
6137 .await
6138 .expect("the hashed-set namespace lock should be created");
6139 let target_err = target_lock
6140 .get_write_lock(Duration::from_millis(50))
6141 .await
6142 .expect_err("the fixed write fence must conflict through the shared clients");
6143 assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. }));
6144
6145 drop(publication_guards);
6146
6147 let cleanup_object = (0..1_000)
6148 .map(|index| format!("decommission-dist-cleanup-{index}.bin"))
6149 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6150 .expect("a cleanup key should hash to the second set namespace");
6151 let source_fence = store
6152 .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref())
6153 .await
6154 .expect("the fixed distributed cleanup fence should be acquired");
6155 assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence");
6156 let source_lock = sets.disk_set[1]
6157 .new_ns_lock("bucket", &cleanup_object)
6158 .await
6159 .expect("the source-set namespace lock should be created");
6160 let source_err = source_lock
6161 .get_read_lock(Duration::from_millis(50))
6162 .await
6163 .expect_err("the fixed write fence must conflict through the shared clients");
6164 assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. }));
6165 }
6166
6167 #[tokio::test]
6168 async fn physical_absence_proof_blocks_decommission_remote_tuple_publication() {
6169 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6170 let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6171 ctx.update_erasure_type(SetupType::DistErasure).await;
6172 let pool_config = sets.endpoints.clone();
6173 let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6174 let bucket = "remote-tuple-publication-fence";
6175 let object = "transitioned.bin";
6176 let encoded = encode_dir_object(object);
6177 let mut proof_opts = ObjectOptions::default();
6178 let proof = store
6179 .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6180 .await
6181 .expect("the all-physical absence proof should acquire every read domain");
6182
6183 let mut publication = Box::pin(store.acquire_data_movement_publication_write_locks(bucket, &encoded, 0, 0, true));
6188 let blocked = tokio::time::timeout(Duration::from_millis(50), &mut publication).await;
6189 assert!(
6190 blocked.is_err(),
6191 "a decommission publisher must remain blocked while the remote-delete absence proof is held"
6192 );
6193
6194 drop(proof);
6195 publication
6196 .await
6197 .expect("the publisher should acquire its write fence after the proof releases");
6198 }
6199
6200 #[cfg(feature = "test-util")]
6201 #[tokio::test]
6202 async fn restored_transitioned_rebalance_revalidates_source_after_recovery_lock() {
6203 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6204 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6205 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6206 let store = Arc::new(
6207 new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], Arc::clone(&ctx)).await,
6208 );
6209 let bucket = "restored-transitioned-rebalance-publication";
6210 let object = "restored.bin";
6211 for set in [&first_set, &second_set] {
6212 set.make_bucket(bucket, &MakeBucketOptions::default())
6213 .await
6214 .expect("publication-race bucket should be created");
6215 }
6216
6217 let tier_name = "RESTORED-REBALANCE";
6218 crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), tier_name).await;
6219 let lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), tier_name)
6220 .await
6221 .expect("mock tier generation should be available");
6222 let backend_identity = lease.backend_identity();
6223 drop(lease);
6224
6225 let mut metadata = HashMap::new();
6226 for (suffix, value) in [
6227 (rustfs_utils::http::SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.to_string()),
6228 (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/restored.bin".to_string()),
6229 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
6230 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-version".to_string()),
6231 (
6232 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6233 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
6234 ),
6235 (
6236 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6237 rustfs_utils::crypto::hex(backend_identity),
6238 ),
6239 ] {
6240 rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, value);
6241 }
6242 metadata.insert(
6243 "x-amz-restore".to_string(),
6244 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
6245 );
6246 let mut body = PutObjReader::from_vec(b"restored local body".to_vec());
6247 first_set
6248 .put_object(
6249 bucket,
6250 object,
6251 &mut body,
6252 &ObjectOptions {
6253 user_defined: metadata,
6254 no_lock: true,
6255 ..Default::default()
6256 },
6257 )
6258 .await
6259 .expect("restored-transitioned source should be written");
6260 let source = first_set
6261 .get_object_info(
6262 bucket,
6263 object,
6264 &ObjectOptions {
6265 include_part_checksums: true,
6266 no_lock: true,
6267 ..Default::default()
6268 },
6269 )
6270 .await
6271 .expect("restored-transitioned source metadata should be readable");
6272 assert_eq!(source.transitioned_object.status, TRANSITION_COMPLETE);
6273 assert!(source.data_dir.is_some(), "fixture must retain restored local data");
6274
6275 let publisher_first = store
6276 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6277 .await
6278 .expect("rebalance should capture an exact publication capability");
6279 let encoded = encode_dir_object(object);
6280 let publisher_guard = publisher_first
6281 .into_commit_guard(1, bucket, &encoded)
6282 .await
6283 .expect("unchanged source should validate at the publication boundary");
6284 let mut publisher_first_proof_opts = ObjectOptions::default();
6285 let mut publisher_first_proof = Box::pin(store.acquire_all_physical_object_read_locks(
6286 "tier_delete_journal_recovery",
6287 bucket,
6288 &encoded,
6289 &mut publisher_first_proof_opts,
6290 ));
6291 assert!(
6292 tokio::time::timeout(Duration::from_millis(50), &mut publisher_first_proof)
6293 .await
6294 .is_err(),
6295 "recovery must wait while a validated publisher owns the commit scope"
6296 );
6297 drop(publisher_guard);
6298 drop(
6299 publisher_first_proof
6300 .await
6301 .expect("recovery proof should acquire after publisher commit scope releases"),
6302 );
6303
6304 let publication = store
6305 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6306 .await
6307 .expect("recovery-first race should capture the old source identity");
6308
6309 let pre_commit_reader = first_set
6311 .new_ns_lock(bucket, &encoded)
6312 .await
6313 .expect("source read lock should be created")
6314 .get_read_lock(Duration::from_millis(50))
6315 .await
6316 .expect("body staging must not be blocked by an early publication write lock");
6317 drop(pre_commit_reader);
6318
6319 first_set
6320 .delete_object(
6321 bucket,
6322 &encoded,
6323 ObjectOptions {
6324 delete_prefix: true,
6325 delete_prefix_object: true,
6326 data_movement: true,
6327 no_lock: true,
6328 ..Default::default()
6329 },
6330 )
6331 .await
6332 .expect("source cleanup should remove the old physical owner");
6333
6334 let mut proof_opts = ObjectOptions::default();
6335 let proof = store
6336 .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6337 .await
6338 .expect("recovery should hold its all-physical absence proof");
6339 let mut commit = Box::pin(publication.into_commit_guard(1, bucket, &encoded));
6340 assert!(
6341 tokio::time::timeout(Duration::from_millis(50), &mut commit).await.is_err(),
6342 "publisher must wait while recovery owns the physical absence proof"
6343 );
6344 drop(proof);
6345 let err = match commit.await {
6346 Ok(_) => panic!("publisher must reject its stale source snapshot after recovery releases"),
6347 Err(err) => err,
6348 };
6349 assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6350 assert!(
6351 second_set
6352 .get_object_info(bucket, object, &ObjectOptions::default())
6353 .await
6354 .is_err(),
6355 "stale publication must not create a target owner"
6356 );
6357 }
6358
6359 #[tokio::test]
6360 async fn data_movement_put_defers_publication_locks_until_body_complete() {
6361 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6362 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6363 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6364 let store =
6365 Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6366 let bucket = "commit-late-data-movement-put";
6367 let object = "slow-body.bin";
6368 for set in [&first_set, &second_set] {
6369 set.make_bucket(bucket, &MakeBucketOptions::default())
6370 .await
6371 .expect("commit-late PUT bucket should be created");
6372 }
6373
6374 let payload = b"data movement body".to_vec();
6375 let mut source_body = PutObjReader::from_vec(payload.clone());
6376 first_set
6377 .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6378 .await
6379 .expect("data-movement source should be written");
6380 let source = first_set
6381 .get_object_info(bucket, object, &ObjectOptions::default())
6382 .await
6383 .expect("data-movement source should be readable");
6384 let publication = store
6385 .acquire_remote_tuple_publication_fence(bucket, 0, &source, true)
6386 .await
6387 .expect("decommission should capture its publication capability");
6388
6389 let payload_len = i64::try_from(payload.len()).expect("test payload length should fit in i64");
6390 let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6391 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6392 let slow_body = CommitLateBodyReader {
6393 inner: Cursor::new(payload),
6394 poll_started: Some(poll_started_tx),
6395 release: Some(release_rx),
6396 };
6397 let target_set = Arc::clone(&store.pools[1]);
6398 let target_opts = ObjectOptions {
6399 data_movement: true,
6400 src_pool_idx: 0,
6401 http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
6402 mod_time: source.mod_time,
6403 preserve_etag: source.etag.clone(),
6404 user_defined: source.user_defined.as_ref().clone(),
6405 ..Default::default()
6406 };
6407 let worker = tokio::spawn(async move {
6408 let hash_reader = rustfs_rio::HashReader::from_stream(slow_body, payload_len, payload_len, None, None, false)
6409 .expect("slow body should produce a valid hash reader");
6410 let mut reader = PutObjReader::new(hash_reader);
6411 target_set
6412 .put_object_with_old_current_size_for_data_movement(bucket, object, &mut reader, &target_opts, publication)
6413 .await
6414 });
6415
6416 tokio::time::timeout(Duration::from_secs(5), poll_started_rx)
6417 .await
6418 .expect("target PUT should start polling its body")
6419 .expect("target PUT should retain the poll observer");
6420 let source_head = tokio::time::timeout(
6421 Duration::from_millis(250),
6422 first_set.get_object_info(bucket, object, &ObjectOptions::default()),
6423 )
6424 .await
6425 .expect("same-key HEAD must not wait behind body staging")
6426 .expect("the source must remain readable during body staging");
6427 assert_eq!(source_head.etag, source.etag);
6428 let target_head = tokio::time::timeout(
6429 Duration::from_millis(250),
6430 second_set.get_object_info(bucket, object, &ObjectOptions::default()),
6431 )
6432 .await
6433 .expect("target HEAD must not wait behind body staging")
6434 .expect_err("the target must remain unpublished before body EOF");
6435 assert!(is_err_object_not_found(&target_head) || is_err_version_not_found(&target_head));
6436
6437 release_tx
6438 .send(())
6439 .expect("target PUT should still be waiting on the body gate");
6440 worker
6441 .await
6442 .expect("target PUT task should join")
6443 .expect("target PUT should commit after body EOF");
6444 second_set
6445 .get_object_info(bucket, object, &ObjectOptions::default())
6446 .await
6447 .expect("target metadata should be visible after the commit boundary");
6448 }
6449
6450 #[test]
6451 fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() {
6452 let protected_suffixes = [
6453 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6454 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
6455 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
6456 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6457 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
6458 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6459 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
6460 rustfs_utils::http::SUFFIX_FREE_VERSION,
6461 rustfs_utils::http::SUFFIX_TIER_FV_ID,
6462 rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
6463 ];
6464 let mut source = HashMap::from([("content-type".to_string(), "application/octet-stream".to_string())]);
6465 for suffix in protected_suffixes {
6466 rustfs_utils::http::metadata_compat::insert_str(&mut source, suffix, format!("source-{suffix}"));
6467 }
6468 source.insert(
6471 format!(
6472 "{}{}",
6473 rustfs_utils::http::MINIO_INTERNAL_PREFIX.to_ascii_uppercase(),
6474 rustfs_utils::http::SUFFIX_TRANSITION_STATUS.to_ascii_uppercase()
6475 ),
6476 "complete".to_string(),
6477 );
6478
6479 let copied = materialized_copy_user_defined(&source);
6480
6481 assert_eq!(copied.get("content-type").map(String::as_str), Some("application/octet-stream"));
6482 for suffix in protected_suffixes {
6483 assert!(
6484 !rustfs_utils::http::metadata_compat::contains_key_str(&copied, suffix),
6485 "materialized copy retained protected internal suffix {suffix}"
6486 );
6487 }
6488 }
6489
6490 #[test]
6491 fn remote_tuple_publication_target_is_exact_legacy_compatible_and_ordinary_zero_cost() {
6492 let mut ordinary_metadata = HashMap::new();
6493 rustfs_utils::http::metadata_compat::insert_str(
6494 &mut ordinary_metadata,
6495 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6496 "malformed-but-irrelevant".to_string(),
6497 );
6498 let ordinary = ObjectInfo {
6499 user_defined: Arc::new(ordinary_metadata),
6500 ..Default::default()
6501 };
6502 assert!(
6503 remote_tuple_publication_target(&ordinary)
6504 .expect("ordinary objects must not parse or lock tier metadata")
6505 .is_none()
6506 );
6507
6508 let identity = [0xabu8; 32];
6509 let mut exact_metadata = HashMap::new();
6510 rustfs_utils::http::metadata_compat::insert_str(
6511 &mut exact_metadata,
6512 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6513 rustfs_utils::crypto::hex(identity),
6514 );
6515 let transitioned = ObjectInfo {
6516 transitioned_object: TransitionedObject {
6517 status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
6518 name: "remote/object".to_string(),
6519 tier: "WARM".to_string(),
6520 ..Default::default()
6521 },
6522 user_defined: Arc::new(exact_metadata),
6523 ..Default::default()
6524 };
6525 let (tier, recorded_identity) = remote_tuple_publication_target(&transitioned)
6526 .expect("exact transitioned metadata should parse")
6527 .expect("transitioned object should require a publication fence");
6528 assert_eq!(tier, "WARM");
6529 assert_eq!(recorded_identity, Some(identity));
6530
6531 let legacy = ObjectInfo {
6532 user_defined: Arc::new(HashMap::new()),
6533 ..transitioned.clone()
6534 };
6535 assert_eq!(
6536 remote_tuple_publication_target(&legacy)
6537 .expect("legacy transitioned metadata should remain compatible")
6538 .map(|(_, identity)| identity),
6539 Some(None)
6540 );
6541
6542 let mut malformed = transitioned;
6543 malformed.transitioned_object.tier.clear();
6544 assert!(remote_tuple_publication_target(&malformed).is_err());
6545 }
6546
6547 #[test]
6548 fn publication_source_match_rejects_new_physical_generation_with_same_logical_identity() {
6549 let expected = ObjectInfo {
6550 data_dir: Some(Uuid::new_v4()),
6551 ..Default::default()
6552 };
6553 assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6554
6555 let current = ObjectInfo {
6556 data_dir: Some(Uuid::new_v4()),
6557 ..expected.clone()
6558 };
6559 assert!(
6560 crate::data_movement::is_equivalent_data_movement_object_identity(&expected, ¤t, true, true),
6561 "the logical target-equivalence check deliberately ignores target data-dir allocation"
6562 );
6563 assert!(
6564 !remote_tuple_publication_object_source_matches(&expected, ¤t),
6565 "publication revalidation must additionally bind the physical source generation"
6566 );
6567 }
6568
6569 #[test]
6570 fn exact_delete_capacity_target_requires_matching_namespace_and_object_identity() {
6571 let source = ObjectInfo {
6572 bucket: "bucket".to_string(),
6573 name: "directory/".to_string(),
6574 version_id: Some(Uuid::from_u128(1)),
6575 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
6576 size: 10,
6577 etag: Some("source-etag".to_string()),
6578 ..Default::default()
6579 };
6580 let target = ObjectInfo {
6581 name: rustfs_utils::path::encode_dir_object(&source.name),
6582 ..source.clone()
6583 };
6584 assert!(ECStore::is_equivalent_decommission_capacity_target(&source, &target));
6585
6586 let mismatched_identity = ObjectInfo {
6587 etag: Some("different-etag".to_string()),
6588 ..target.clone()
6589 };
6590 assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &mismatched_identity));
6591
6592 let wrong_bucket = ObjectInfo {
6593 bucket: "other-bucket".to_string(),
6594 ..target
6595 };
6596 assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &wrong_bucket));
6597 }
6598
6599 fn publication_part(number: usize) -> ObjectPartInfo {
6600 ObjectPartInfo {
6601 number,
6602 size: 100 + number,
6603 actual_size: i64::try_from(200 + number).expect("test part size should fit in i64"),
6604 etag: format!("part-etag-{number}"),
6605 ..Default::default()
6606 }
6607 }
6608
6609 fn publication_source(parts: Vec<ObjectPartInfo>) -> ObjectInfo {
6610 ObjectInfo {
6611 data_dir: Some(Uuid::new_v4()),
6612 version_id: Some(Uuid::new_v4()),
6613 size: 4096,
6614 actual_size: 4096,
6615 etag: Some("object-etag".to_string()),
6616 checksum: Some(Bytes::from_static(b"object-checksum")),
6617 mod_time: Some(time::OffsetDateTime::UNIX_EPOCH),
6618 parts: Arc::new(parts),
6619 ..Default::default()
6620 }
6621 }
6622
6623 #[test]
6624 fn publication_parts_match_is_order_independent_and_bijective() {
6625 let ordered = vec![publication_part(1), publication_part(2), publication_part(3)];
6626 let mut reversed = ordered.clone();
6627 reversed.reverse();
6628
6629 assert!(remote_tuple_publication_parts_match(&[], &[]));
6630 assert!(remote_tuple_publication_parts_match(&ordered, &ordered));
6631 assert!(remote_tuple_publication_parts_match(&ordered, &reversed));
6632 assert!(!remote_tuple_publication_parts_match(&ordered[..2], &ordered));
6633
6634 let expected_duplicate = vec![publication_part(1), publication_part(1)];
6635 let current_unique = vec![publication_part(1), publication_part(2)];
6636 assert!(
6637 !remote_tuple_publication_parts_match(&expected_duplicate, ¤t_unique),
6638 "two expected entries must not reuse the same current part"
6639 );
6640 assert!(!remote_tuple_publication_parts_match(¤t_unique, &expected_duplicate));
6641 assert!(!remote_tuple_publication_parts_match(&expected_duplicate, &expected_duplicate));
6642 }
6643
6644 #[test]
6645 fn publication_parts_match_preserves_exact_part_identity_contract() {
6646 let expected = vec![publication_part(1)];
6647
6648 let mut different_number = expected.clone();
6649 different_number[0].number = 2;
6650 assert!(!remote_tuple_publication_parts_match(&expected, &different_number));
6651
6652 let mut different_size = expected.clone();
6653 different_size[0].size += 1;
6654 assert!(!remote_tuple_publication_parts_match(&expected, &different_size));
6655
6656 let mut different_actual_size = expected.clone();
6657 different_actual_size[0].actual_size += 1;
6658 assert!(!remote_tuple_publication_parts_match(&expected, &different_actual_size));
6659
6660 let mut different_etag = expected.clone();
6661 different_etag[0].etag.push_str("-changed");
6662 assert!(!remote_tuple_publication_parts_match(&expected, &different_etag));
6663
6664 let mut ignored_fields = expected.clone();
6665 ignored_fields[0].index = Some(Bytes::from_static(b"different-index"));
6666 ignored_fields[0].checksums = Some(HashMap::from([("CRC32C".to_string(), "different".to_string())]));
6667 ignored_fields[0].mod_time = Some(time::OffsetDateTime::UNIX_EPOCH);
6668 assert!(
6669 remote_tuple_publication_parts_match(&expected, &ignored_fields),
6670 "the publication fence must retain its existing checksum/index/mod-time compatibility contract"
6671 );
6672
6673 let zero_actual_size = vec![ObjectPartInfo {
6674 actual_size: 0,
6675 ..publication_part(1)
6676 }];
6677 assert!(
6678 !remote_tuple_publication_parts_match(&zero_actual_size, &expected),
6679 "the publication fence compares raw part actual sizes without the broader comparator's fallback"
6680 );
6681 }
6682
6683 #[test]
6684 fn publication_source_match_rejects_object_etag_and_identity_mutations() {
6685 let expected = publication_source(vec![publication_part(1)]);
6686 assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6687
6688 let mut current = expected.clone();
6689 current.etag = Some("changed-object-etag".to_string());
6690 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6691
6692 let mut current = expected.clone();
6693 current.checksum = Some(Bytes::from_static(b"changed-checksum"));
6694 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6695
6696 let mut current = expected.clone();
6697 current.actual_size += 1;
6698 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6699
6700 let mut current = expected.clone();
6701 current.data_dir = Some(Uuid::new_v4());
6702 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6703
6704 let mut current = expected.clone();
6705 current.version_id = Some(Uuid::new_v4());
6706 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6707
6708 let mut current = expected.clone();
6709 current.mod_time = current.mod_time.map(|mod_time| mod_time + time::Duration::SECOND);
6710 assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
6711 }
6712
6713 #[test]
6714 fn publication_source_match_preserves_effective_actual_size_compatibility() {
6715 let expected = publication_source(vec![publication_part(1)]);
6716 let current = ObjectInfo {
6717 actual_size: 0,
6718 ..expected.clone()
6719 };
6720
6721 assert_eq!(
6722 expected.get_actual_size().expect("expected size should be valid"),
6723 current.get_actual_size().expect("current size should be valid")
6724 );
6725 assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
6726 }
6727
6728 #[test]
6729 fn publication_source_match_rejects_duplicate_parts_through_the_full_fence() {
6730 let expected = publication_source(vec![publication_part(1), publication_part(1)]);
6731 let current = ObjectInfo {
6732 parts: Arc::new(vec![publication_part(1), publication_part(2)]),
6733 ..expected.clone()
6734 };
6735
6736 assert!(
6737 !remote_tuple_publication_object_source_matches(&expected, ¤t),
6738 "the full source fence must reject the original non-bijective false-positive"
6739 );
6740 }
6741
6742 #[test]
6743 fn publication_source_match_handles_10_000_reversed_parts_without_payload_cloning() {
6744 let parts = Arc::new((1..=10_000).map(publication_part).collect::<Vec<_>>());
6745 let mut reversed = parts.as_ref().clone();
6746 reversed.reverse();
6747 let expected = publication_source(Vec::new());
6748 let expected = ObjectInfo {
6749 parts: Arc::clone(&parts),
6750 ..expected
6751 };
6752 let current = ObjectInfo {
6753 parts: Arc::new(reversed),
6754 ..expected.clone()
6755 };
6756 let expected_parts = Arc::clone(&expected.parts);
6757
6758 assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
6759 assert!(Arc::ptr_eq(&expected.parts, &expected_parts));
6760 }
6761
6762 #[tokio::test]
6763 async fn publication_commit_guard_rejects_an_etag_only_source_change() {
6764 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6765 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6766 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6767 let store =
6768 Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6769 let bucket = "publication-etag-source-change";
6770 let object = "source.bin";
6771 for set in [&first_set, &second_set] {
6772 set.make_bucket(bucket, &MakeBucketOptions::default())
6773 .await
6774 .expect("publication ETag test bucket should be created");
6775 }
6776
6777 let mut source_body = PutObjReader::from_vec(b"source body".to_vec());
6778 first_set
6779 .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6780 .await
6781 .expect("publication source should be written");
6782 let source = first_set
6783 .get_object_info(
6784 bucket,
6785 object,
6786 &ObjectOptions {
6787 include_part_checksums: true,
6788 ..Default::default()
6789 },
6790 )
6791 .await
6792 .expect("publication source should be readable");
6793 let publication = store
6794 .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6795 .await
6796 .expect("the source snapshot should produce a publication capability");
6797
6798 let changed = first_set
6799 .put_object_metadata(
6800 bucket,
6801 object,
6802 &ObjectOptions {
6803 eval_metadata: Some(HashMap::from([("etag".to_string(), "changed-etag".to_string())])),
6804 ..Default::default()
6805 },
6806 )
6807 .await
6808 .expect("the source ETag should be updated in place");
6809 assert_ne!(source.etag, changed.etag);
6810 let unchanged_except_etag = ObjectInfo {
6811 etag: source.etag.clone(),
6812 ..changed.clone()
6813 };
6814 assert!(
6815 remote_tuple_publication_object_source_matches(&source, &unchanged_except_etag),
6816 "the persisted source update fixture must differ only by object ETag"
6817 );
6818
6819 let encoded = encode_dir_object(object);
6820 let err = match publication.into_commit_guard(1, bucket, &encoded).await {
6821 Ok(_) => panic!("the publication guard must reject an ETag-only source change"),
6822 Err(err) => err,
6823 };
6824 assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6825 assert!(
6826 second_set
6827 .get_object_info(bucket, object, &ObjectOptions::default())
6828 .await
6829 .is_err(),
6830 "a rejected publication must not create a target object"
6831 );
6832 }
6833
6834 #[tokio::test]
6835 async fn generic_data_movement_put_rejects_transition_ownership_without_capability() {
6836 let (_dirs, set) = make_local_set_disks(4, 2).await;
6837 let bucket = "unfenced-transition-publication";
6838 set.make_bucket(bucket, &MakeBucketOptions::default())
6839 .await
6840 .expect("unfenced publication bucket should be created");
6841 let mut metadata = HashMap::new();
6842 rustfs_utils::http::metadata_compat::insert_str(
6843 &mut metadata,
6844 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6845 TRANSITION_COMPLETE.to_string(),
6846 );
6847 let mut body = PutObjReader::from_vec(b"must not publish".to_vec());
6848 let err = set
6849 .put_object(
6850 bucket,
6851 "unfenced.bin",
6852 &mut body,
6853 &ObjectOptions {
6854 data_movement: true,
6855 user_defined: metadata,
6856 ..Default::default()
6857 },
6858 )
6859 .await
6860 .expect_err("transition ownership must require the typed publication entry point");
6861 assert!(err.to_string().contains("publication capability"));
6862 }
6863
6864 #[test]
6865 fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() {
6866 let nil = Uuid::nil();
6867 for actual in [None, Some(nil)] {
6868 assert!(select_snapshot_version_matches(actual, "null"));
6869 assert!(select_snapshot_version_matches(actual, "NULL"));
6870 assert!(select_snapshot_version_matches(actual, &nil.to_string()));
6871 }
6872
6873 let version = Uuid::new_v4();
6874 assert!(select_snapshot_version_matches(Some(version), &version.to_string().to_uppercase()));
6875 assert!(!select_snapshot_version_matches(Some(version), "null"));
6876 assert!(!select_snapshot_version_matches(Some(version), "not-a-version"));
6877 }
6878
6879 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6880 async fn select_snapshot_reader_rolls_back_bytes_when_lock_is_lost_during_poll() {
6881 let (lease, signal, client) = refresh_failure_test_lease("select-snapshot-lock-loss").await;
6882
6883 let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6884 let (resume_tx, resume_rx) = std::sync::mpsc::channel();
6885 let release_client = Arc::clone(&client);
6886 let release_lease = Arc::clone(&lease);
6887 let release_signal = Arc::clone(&signal);
6888 let release_task = tokio::spawn(async move {
6889 poll_started_rx.await.expect("reader poll should start");
6890 release_client.reject_refreshes();
6891 release_client
6892 .wait_for_rejected_refresh(Duration::from_secs(5))
6893 .await
6894 .expect("refresh rejection should be observed");
6895 tokio::time::timeout(Duration::from_secs(5), release_signal.notified())
6896 .await
6897 .expect("heartbeat should observe the rejected refresh");
6898 tokio::time::timeout(Duration::from_secs(5), async {
6899 while !release_lease.is_lost() {
6900 tokio::task::yield_now().await;
6901 }
6902 })
6903 .await
6904 .expect("snapshot monitor should publish lock loss");
6905 resume_tx.send(()).expect("reader poll should still be waiting");
6906 });
6907
6908 let mut reader = SelectObjectSnapshotReader {
6909 inner: Box::new(WaitForLockLossReader {
6910 inner: Cursor::new(b"must-not-escape".to_vec()),
6911 poll_started: Some(poll_started_tx),
6912 resume: ParkingMutex::new(resume_rx),
6913 }),
6914 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6915 lease,
6916 };
6917 let mut output = Vec::new();
6918 let error = reader
6919 .read_to_end(&mut output)
6920 .await
6921 .expect_err("a read that loses its lease must fail");
6922 release_task.await.expect("backend release task should not panic");
6923
6924 assert!(signal.is_lost(), "heartbeat should report the rejected refresh");
6925 assert!(output.is_empty(), "bytes read during the failed poll must be rolled back");
6926 assert_eq!(error.kind(), std::io::ErrorKind::Other);
6927 }
6928
6929 #[tokio::test(flavor = "current_thread")]
6930 async fn select_snapshot_reader_checks_guards_before_monitor_runs() {
6931 let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await;
6932 client.reject_refreshes();
6933 client
6934 .wait_for_rejected_refresh(Duration::from_secs(5))
6935 .await
6936 .expect("refresh rejection should be observed");
6937 tokio::time::timeout(Duration::from_secs(5), signal.notified())
6938 .await
6939 .expect("heartbeat should observe the rejected refresh");
6940
6941 let lease = Arc::new(SelectObjectSnapshotLease::new(vec![guard]));
6942 assert!(!lease.is_lost(), "current-thread monitor must not run before the synchronous read");
6943 let mut reader = SelectObjectSnapshotReader {
6944 inner: Box::new(Cursor::new(b"old-generation".to_vec())),
6945 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6946 lease,
6947 };
6948 let mut output = Vec::new();
6949
6950 let error = reader
6951 .read_to_end(&mut output)
6952 .await
6953 .expect_err("EOF fence must reject a lease lost before its monitor is scheduled");
6954
6955 assert!(output.is_empty(), "bytes from a known-lost snapshot must not escape");
6956 assert_eq!(error.kind(), std::io::ErrorKind::Other);
6957 }
6958
6959 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6960 async fn select_snapshot_lock_loss_is_broadcast_to_remaining_pending_readers() {
6961 let (first_guard, first_signal, first_client) = refresh_failure_test_guard("select-snapshot-pending-first-lock").await;
6962 let (second_guard, second_signal, second_client) =
6963 refresh_failure_test_guard("select-snapshot-pending-second-lock").await;
6964 let lease = Arc::new(SelectObjectSnapshotLease::new(vec![first_guard, second_guard]));
6965 let dropped_reader = SelectObjectSnapshotReader {
6966 inner: Box::new(PermanentlyPendingReader { poll_started: None }),
6967 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6968 lease: Arc::clone(&lease),
6969 };
6970 drop(dropped_reader);
6971
6972 let (first_started_tx, first_started_rx) = tokio::sync::oneshot::channel();
6973 let (second_started_tx, second_started_rx) = tokio::sync::oneshot::channel();
6974 let mut first_reader = SelectObjectSnapshotReader {
6975 inner: Box::new(PermanentlyPendingReader {
6976 poll_started: Some(first_started_tx),
6977 }),
6978 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6979 lease: Arc::clone(&lease),
6980 };
6981 let mut second_reader = SelectObjectSnapshotReader {
6982 inner: Box::new(PermanentlyPendingReader {
6983 poll_started: Some(second_started_tx),
6984 }),
6985 lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6986 lease,
6987 };
6988 let first_read_task = tokio::spawn(async move {
6989 let mut byte = [0_u8; 1];
6990 first_reader.read(&mut byte).await
6991 });
6992 let second_read_task = tokio::spawn(async move {
6993 let mut byte = [0_u8; 1];
6994 second_reader.read(&mut byte).await
6995 });
6996
6997 first_started_rx.await.expect("first inner reader should reach Poll::Pending");
6998 second_started_rx
6999 .await
7000 .expect("second inner reader should reach Poll::Pending");
7001 second_client.reject_refreshes();
7002 second_client
7003 .wait_for_rejected_refresh(Duration::from_secs(5))
7004 .await
7005 .expect("refresh rejection should be observed");
7006 let (first_result, second_result) = tokio::join!(
7007 tokio::time::timeout(Duration::from_secs(5), first_read_task),
7008 tokio::time::timeout(Duration::from_secs(5), second_read_task),
7009 );
7010 for result in [first_result, second_result] {
7011 let error = result
7012 .expect("lock loss should wake every reader whose inner I/O remains pending")
7013 .expect("reader task should not panic")
7014 .expect_err("lost snapshot lease must fail every pending read");
7015 assert_eq!(error.kind(), std::io::ErrorKind::Other);
7016 }
7017
7018 assert!(!first_client.refreshes_rejected());
7019 assert!(!first_signal.is_lost());
7020 assert!(second_signal.is_lost());
7021 }
7022
7023 #[test]
7024 fn tier_delete_entry_is_prepared_and_prefix_legacy_fails_closed() {
7025 let identity = [9_u8; 32];
7026 let mut metadata = HashMap::new();
7027 rustfs_utils::http::metadata_compat::insert_str(
7028 &mut metadata,
7029 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7030 rustfs_utils::crypto::hex(identity),
7031 );
7032 let version_id = Uuid::from_u128(1);
7033 let data_dir = Uuid::from_u128(2);
7034 let source = ObjectInfo {
7035 bucket: "bucket".to_string(),
7036 name: "object".to_string(),
7037 version_id: Some(version_id),
7038 data_dir: Some(data_dir),
7039 user_defined: Arc::new(metadata),
7040 transitioned_object: TransitionedObject {
7041 name: "remote/object".to_string(),
7042 version_id: "remote-version".to_string(),
7043 tier: "WARM".to_string(),
7044 status: TRANSITION_COMPLETE.to_string(),
7045 ..Default::default()
7046 },
7047 transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
7048 ..Default::default()
7049 };
7050 let entry = build_tier_delete_journal_entry(
7051 "bucket",
7052 "object",
7053 &ObjectOptions {
7054 version_id: Some(version_id.to_string()),
7055 versioned: true,
7056 ..Default::default()
7057 },
7058 &source,
7059 )
7060 .expect("transition source should produce a journal entry")
7061 .expect("completed transition should be journaled");
7062
7063 assert_eq!(entry.state, TierDeleteJournalState::Prepared);
7064 assert_eq!(entry.backend_identity, Some(identity));
7065 let data_dir_string = data_dir.to_string();
7066 assert_eq!(
7067 entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
7068 Some(data_dir_string.as_str())
7069 );
7070
7071 let mut unknown = source.clone();
7072 unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7073 assert!(
7074 build_tier_delete_journal_entry(
7075 "bucket",
7076 "object",
7077 &ObjectOptions {
7078 delete_prefix: true,
7079 versioned: true,
7080 ..Default::default()
7081 },
7082 &unknown,
7083 )
7084 .expect_err("recursive prefix delete must fail closed for legacy Unknown metadata")
7085 .to_string()
7086 .contains("legacy Unknown")
7087 );
7088
7089 assert!(
7090 build_tier_delete_journal_entry(
7091 "bucket",
7092 "object",
7093 &ObjectOptions {
7094 delete_prefix: true,
7095 delete_prefix_object: true,
7096 versioned: true,
7097 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
7098 version_id: Some(version_id),
7099 delete_marker: false,
7100 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
7101 rule_id: "delete-all".to_string(),
7102 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
7103 }),
7104 ..Default::default()
7105 },
7106 &unknown,
7107 )
7108 .expect("lifecycle exact-object delete-all may use its free-version fallback")
7109 .is_none()
7110 );
7111
7112 let mut identity_less = source;
7113 identity_less.user_defined = Arc::new(HashMap::new());
7114 assert!(
7115 build_tier_delete_journal_entry(
7116 "bucket",
7117 "object",
7118 &ObjectOptions {
7119 delete_prefix: true,
7120 versioned: true,
7121 ..Default::default()
7122 },
7123 &identity_less,
7124 )
7125 .expect_err("recursive prefix delete must fail closed without destination identity")
7126 .to_string()
7127 .contains("requires a stable transitioned source")
7128 );
7129 }
7130
7131 #[test]
7132 fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
7133 let cancelled = Error::OperationCanceled;
7134 assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
7135 assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
7136 assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
7137 assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
7138 }
7139
7140 #[test]
7141 fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
7142 let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
7143 .expect_err("a real walk error must not be hidden by earlier cancellation");
7144 assert_eq!(err, StorageError::FileAccessDenied);
7145
7146 assert_eq!(
7147 combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
7148 .expect_err("cancellation must remain visible when there is no real error"),
7149 Error::OperationCanceled
7150 );
7151 }
7152
7153 impl Drop for BodyCacheHookGuard {
7154 fn drop(&mut self) {
7155 clear_get_object_body_cache_hook();
7156 }
7157 }
7158
7159 #[test]
7160 fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
7161 let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
7162 assert_eq!(target, None);
7163 }
7164
7165 #[test]
7166 fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
7167 let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
7168 let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
7169 assert_eq!(target, None);
7170 }
7171
7172 #[test]
7173 fn non_delete_marker_data_movement_keeps_existing_pool() {
7174 let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
7175 assert_eq!(target, Some(0));
7176 }
7177
7178 #[test]
7179 fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
7180 let version_id = Uuid::nil();
7181 let mod_time = OffsetDateTime::UNIX_EPOCH;
7182 let source = ObjectInfo {
7183 version_id: Some(version_id),
7184 delete_marker: true,
7185 mod_time: Some(mod_time),
7186 ..Default::default()
7187 };
7188 let target = source.clone();
7189
7190 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7191
7192 let mismatched = ObjectInfo {
7193 mod_time: Some(mod_time + Duration::from_secs(1)),
7194 ..target
7195 };
7196 assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
7197 }
7198
7199 #[test]
7200 fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
7201 let mut source = ObjectInfo {
7202 version_id: Some(Uuid::from_u128(1)),
7203 delete_marker: true,
7204 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7205 ..Default::default()
7206 };
7207 rustfs_utils::http::insert_str(
7208 Arc::make_mut(&mut source.user_defined),
7209 rustfs_utils::http::SUFFIX_TIER_FV_ID,
7210 Uuid::from_u128(2).to_string(),
7211 );
7212 let mut target = source.clone();
7213 rustfs_utils::http::insert_str(
7214 Arc::make_mut(&mut target.user_defined),
7215 rustfs_utils::http::SUFFIX_TIER_FV_ID,
7216 Uuid::from_u128(3).to_string(),
7217 );
7218
7219 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7220
7221 Arc::make_mut(&mut target.user_defined).insert(
7222 format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
7223 Uuid::from_u128(4).to_string(),
7224 );
7225 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7226 }
7227
7228 #[test]
7229 fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
7230 let key = format!(
7231 "{}{}",
7232 rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7233 rustfs_utils::http::SUFFIX_REPLICATION_STATUS
7234 );
7235 let timestamp_key = format!(
7236 "{}{}",
7237 rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7238 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
7239 );
7240 let source = ObjectInfo {
7241 version_id: Some(Uuid::from_u128(1)),
7242 delete_marker: true,
7243 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7244 user_defined: Arc::new(HashMap::from([
7245 (key.clone(), "arn=COMPLETED;".to_string()),
7246 (timestamp_key, "1970-01-01T00:00:01Z".to_string()),
7247 ])),
7248 ..Default::default()
7249 };
7250 let mut target = source.clone();
7251 rustfs_utils::http::insert_str(
7252 Arc::make_mut(&mut target.user_defined),
7253 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7254 "arn=COMPLETED;".to_string(),
7255 );
7256 rustfs_utils::http::insert_str(
7257 Arc::make_mut(&mut target.user_defined),
7258 rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
7259 (OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
7260 );
7261 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7262
7263 Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
7264 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7265 }
7266
7267 #[test]
7268 fn equivalent_data_movement_delete_marker_ignores_target_bucket_incarnation_fence() {
7269 let source = ObjectInfo {
7270 version_id: Some(Uuid::from_u128(1)),
7271 delete_marker: true,
7272 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7273 ..Default::default()
7274 };
7275 let mut target = source.clone();
7276 rustfs_utils::http::insert_str(
7277 Arc::make_mut(&mut target.user_defined),
7278 rustfs_utils::http::SUFFIX_BUCKET_INCARNATION_ID,
7279 Uuid::from_u128(2).to_string(),
7280 );
7281
7282 assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7283 }
7284
7285 #[test]
7286 fn data_movement_delete_marker_source_requires_persisted_mod_time() {
7287 let source = ObjectInfo {
7288 delete_marker: true,
7289 ..Default::default()
7290 };
7291 assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7292
7293 let source = ObjectInfo {
7294 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7295 ..source
7296 };
7297 assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
7298 assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7299 }
7300
7301 #[test]
7302 fn data_movement_delete_marker_uses_current_source_replication_state() {
7303 let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
7304 let timestamp = expected_timestamp.to_string();
7305 let mut metadata = HashMap::new();
7306 rustfs_utils::http::insert_str(
7307 &mut metadata,
7308 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7309 ReplicationStatusType::Replica.to_string(),
7310 );
7311 rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
7312 rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
7313 rustfs_utils::http::insert_str(
7314 &mut metadata,
7315 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7316 "arn=COMPLETED;".to_string(),
7317 );
7318 rustfs_utils::http::insert_str(
7319 &mut metadata,
7320 &format!(
7321 "{}{}",
7322 rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
7323 "arn:minio:replication::TenantA:bucket"
7324 ),
7325 "reset-id".to_string(),
7326 );
7327 rustfs_utils::http::insert_str(
7328 &mut metadata,
7329 &format!(
7330 "{}{}",
7331 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
7332 "arn:minio:replication::TenantA:bucket"
7333 ),
7334 "target-version".to_string(),
7335 );
7336 let source = ObjectInfo {
7337 delete_marker: true,
7338 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7339 replication_status_internal: Some("arn=COMPLETED;".to_string()),
7340 replication_decision: "arn=replicate;".to_string(),
7341 user_defined: Arc::new(metadata),
7342 ..Default::default()
7343 };
7344 let opts = ObjectOptions {
7345 mod_time: source.mod_time,
7346 delete_replication: Some(ReplicationState {
7347 replication_status_internal: Some("arn=PENDING;".to_string()),
7348 ..Default::default()
7349 }),
7350 ..Default::default()
7351 };
7352
7353 let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
7354 let state = target_opts.delete_replication.as_ref().expect("current replication state");
7355 assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
7356 assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
7357 assert_eq!(state.replica_timestamp, Some(expected_timestamp));
7358 assert_eq!(state.replication_timestamp, state.replica_timestamp);
7359 assert_eq!(state.replicate_decision_str, "arn=replicate;");
7360 assert_eq!(
7361 state
7362 .reset_statuses_map
7363 .get("arn:minio:replication::TenantA:bucket")
7364 .map(String::as_str),
7365 Some("reset-id")
7366 );
7367 assert_eq!(
7368 state
7369 .target_delete_marker_version_ids
7370 .get("arn:minio:replication::TenantA:bucket")
7371 .map(String::as_str),
7372 Some("target-version")
7373 );
7374 }
7375
7376 #[test]
7377 fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
7378 let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
7379 let mut malformed = HashMap::new();
7380 rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
7381 let malformed_source = ObjectInfo {
7382 delete_marker: true,
7383 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7384 user_defined: Arc::new(malformed),
7385 ..Default::default()
7386 };
7387 assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
7388
7389 let mut conflicted = HashMap::new();
7390 let suffix = format!(
7391 "{}arn:minio:replication::target:bucket",
7392 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7393 );
7394 rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
7395 conflicted.insert(
7396 format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7397 "target-version-b".to_string(),
7398 );
7399 let conflicted_source = ObjectInfo {
7400 user_defined: Arc::new(conflicted),
7401 ..malformed_source.clone()
7402 };
7403 assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
7404
7405 let mut over_cap = HashMap::new();
7406 for index in 0..=1_000 {
7407 let suffix = format!(
7408 "{}arn:minio:replication::target:bucket-{index}",
7409 rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7410 );
7411 rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
7412 }
7413 let over_cap_source = ObjectInfo {
7414 user_defined: Arc::new(over_cap),
7415 ..malformed_source
7416 };
7417 assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
7418 }
7419
7420 #[test]
7421 fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
7422 let mut source_metadata = HashMap::new();
7423 rustfs_utils::http::insert_str(
7424 &mut source_metadata,
7425 rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7426 ReplicationStatusType::Replica.to_string(),
7427 );
7428 rustfs_utils::http::insert_str(
7429 &mut source_metadata,
7430 rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7431 "arn=COMPLETED;".to_string(),
7432 );
7433 let source = ObjectInfo {
7434 delete_marker: true,
7435 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7436 replication_status_internal: Some("arn=COMPLETED;".to_string()),
7437 user_defined: Arc::new(source_metadata),
7438 ..Default::default()
7439 };
7440
7441 let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
7442 .expect("legacy status-only metadata should remain migratable");
7443 let state = target_opts
7444 .delete_replication
7445 .expect("replication state should be reconstructed");
7446 assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7447 assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7448
7449 let mut target_metadata = (*source.user_defined).clone();
7450 let epoch = OffsetDateTime::UNIX_EPOCH
7451 .format(&time::format_description::well_known::Rfc3339)
7452 .unwrap();
7453 rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
7454 rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
7455 assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
7456 }
7457
7458 #[test]
7459 fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
7460 let version_id = Uuid::nil();
7461 let mod_time = OffsetDateTime::UNIX_EPOCH;
7462 let source = ObjectInfo {
7463 version_id: Some(version_id),
7464 delete_marker: true,
7465 mod_time: Some(mod_time),
7466 user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
7467 replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7468 version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7469 ..Default::default()
7470 };
7471
7472 let mut target = source.clone();
7473 target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
7474 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7475
7476 let mut target = source.clone();
7477 target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7478 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7479
7480 let mut target = source.clone();
7481 target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7482 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7483 }
7484
7485 #[test]
7486 fn equivalent_data_movement_delete_marker_rejects_live_object() {
7487 let source = ObjectInfo {
7488 delete_marker: true,
7489 ..Default::default()
7490 };
7491 let target = ObjectInfo {
7492 delete_marker: false,
7493 ..source.clone()
7494 };
7495
7496 assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7497 }
7498
7499 #[test]
7500 fn data_movement_delete_marker_resume_accepts_equivalent_target() {
7501 let source = ObjectInfo {
7502 version_id: Some(Uuid::nil()),
7503 delete_marker: true,
7504 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7505 ..Default::default()
7506 };
7507
7508 let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
7509 .expect("equivalent delete marker target should be evaluated");
7510
7511 assert!(should_resume);
7512 }
7513
7514 #[test]
7515 fn data_movement_delete_marker_resume_rejects_source_pool_target() {
7516 let source = ObjectInfo {
7517 version_id: Some(Uuid::nil()),
7518 delete_marker: true,
7519 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7520 ..Default::default()
7521 };
7522
7523 let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
7524 .expect("source-pool target should be rejected before target lookup");
7525
7526 assert!(!should_resume);
7527 }
7528
7529 #[test]
7530 fn data_movement_resume_target_prefers_selected_non_source_pool() {
7531 let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
7532 assert_eq!(target_pool_idx, 2);
7533 }
7534
7535 #[test]
7536 fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
7537 let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
7538 assert_eq!(target_pool_idx, 3);
7539 assert!(should_check_data_movement_resume_target(1, target_pool_idx));
7540 }
7541
7542 #[test]
7543 fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
7544 let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
7545 assert_eq!(target_pool_idx, 1);
7546 }
7547
7548 #[test]
7549 fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
7550 let source = ObjectInfo {
7551 delete_marker: true,
7552 ..Default::default()
7553 };
7554 let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
7555
7556 assert!(matches!(result, Err(Error::SlowDown)));
7557 }
7558
7559 fn tiered_equivalence_source() -> FileInfo {
7560 let version_id = Uuid::nil();
7561 let transition_version_id = Uuid::new_v4();
7562 let mod_time = OffsetDateTime::UNIX_EPOCH;
7563
7564 FileInfo {
7565 version_id: Some(version_id),
7566 size: 1024,
7567 mod_time: Some(mod_time),
7568 checksum: Some(Bytes::from_static(b"checksum")),
7569 transition_status: TRANSITION_COMPLETE.to_string(),
7570 transitioned_objname: "remote/object".to_string(),
7571 transition_tier: "WARM".to_string(),
7572 transition_version_id: Some(transition_version_id),
7573 replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
7574 replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7575 targets: replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
7576 version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7577 purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
7578 ..Default::default()
7579 })),
7580 metadata: HashMap::from([
7581 ("etag".to_string(), "etag-value".to_string()),
7582 ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
7583 (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
7584 ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
7585 ]),
7586 ..Default::default()
7587 }
7588 }
7589
7590 fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
7591 ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
7592 }
7593
7594 #[test]
7595 fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
7596 let source = tiered_equivalence_source();
7597 let target = tiered_equivalence_target(&source);
7598
7599 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7600 }
7601
7602 #[test]
7603 fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
7604 let source = tiered_equivalence_source();
7605 assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
7606
7607 let mut changed_remote = source.clone();
7608 changed_remote.transitioned_objname = "remote/replaced".to_string();
7609 assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
7610
7611 let mut changed_backend = source.clone();
7612 rustfs_utils::http::metadata_compat::insert_str(
7613 &mut changed_backend.metadata,
7614 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7615 rustfs_utils::crypto::hex([9; 32]),
7616 );
7617 assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
7618 }
7619
7620 #[test]
7621 fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
7622 let mut compressed = tiered_equivalence_source();
7623 compressed.size = 600;
7624 rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
7625 rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
7626 let compressed_target = tiered_equivalence_target(&compressed);
7627 assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
7628
7629 let mut encrypted = tiered_equivalence_source();
7630 encrypted.size = 640;
7631 encrypted.metadata.insert(
7632 rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
7633 "key-id".to_string(),
7634 );
7635 encrypted.metadata.insert(
7636 rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
7637 "1024".to_string(),
7638 );
7639 let encrypted_target = tiered_equivalence_target(&encrypted);
7640 assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
7641 }
7642
7643 #[test]
7644 fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
7645 let mut source = tiered_equivalence_source();
7646 let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
7647 source.metadata.insert(
7648 format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7649 source.transition_tier.clone(),
7650 );
7651 let mut target = tiered_equivalence_target(&source);
7652 Arc::make_mut(&mut target.user_defined)
7653 .insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
7654
7655 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7656 }
7657
7658 #[test]
7659 fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
7660 let mut source = tiered_equivalence_source();
7661 source.parts = vec![rustfs_filemeta::ObjectPartInfo {
7662 number: 1,
7663 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
7664 checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
7665 ..Default::default()
7666 }];
7667 rustfs_utils::http::insert_str(
7668 &mut source.metadata,
7669 rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
7670 r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
7671 );
7672 let mut target = tiered_equivalence_target(&source);
7673 Arc::make_mut(&mut target.parts)[0].mod_time = None;
7674 assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7675
7676 let mut missing = target;
7677 Arc::make_mut(&mut missing.parts)[0].checksums = None;
7678 assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
7679 }
7680
7681 #[test]
7682 fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
7683 let source = tiered_equivalence_source();
7684 let mut target = tiered_equivalence_target(&source);
7685 target.transitioned_object.name = "remote/target".to_string();
7686
7687 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7688 }
7689
7690 #[test]
7691 fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
7692 let mut source = tiered_equivalence_source();
7693 source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
7694 let mut target = tiered_equivalence_target(&source);
7695 target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7696
7697 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7698 }
7699
7700 #[test]
7701 fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
7702 let source = tiered_equivalence_source();
7703 let mut target = tiered_equivalence_target(&source);
7704 target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));
7705
7706 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7707 }
7708
7709 #[test]
7710 fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
7711 let source = tiered_equivalence_source();
7712 let mut target = tiered_equivalence_target(&source);
7713 target.user_tags = Arc::new("tag=target".to_string());
7714
7715 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7716 }
7717
7718 #[test]
7719 fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
7720 let source = tiered_equivalence_source();
7721 let mut target = tiered_equivalence_target(&source);
7722 target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7723 target.replication_status = ReplicationStatusType::Failed;
7724
7725 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7726 }
7727
7728 #[test]
7729 fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
7730 let source = tiered_equivalence_source();
7731 let mut target = tiered_equivalence_target(&source);
7732 target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7733 target.version_purge_status = VersionPurgeStatusType::Complete;
7734
7735 assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7736 }
7737
7738 #[test]
7739 fn data_movement_tiered_resume_accepts_equivalent_target() {
7740 let source = tiered_equivalence_source();
7741 let target = tiered_equivalence_target(&source);
7742
7743 let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
7744 .expect("equivalent tiered target should be evaluated");
7745
7746 assert!(should_resume);
7747 }
7748
7749 #[test]
7750 fn data_movement_tiered_resume_rejects_source_pool_target() {
7751 let source = tiered_equivalence_source();
7752 let target = tiered_equivalence_target(&source);
7753
7754 let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
7755 .expect("source-pool target should be rejected before target lookup");
7756
7757 assert!(!should_resume);
7758 }
7759
7760 #[test]
7761 fn data_movement_tiered_resume_rejects_missing_target() {
7762 let source = FileInfo {
7763 version_id: Some(Uuid::nil()),
7764 size: 1024,
7765 ..Default::default()
7766 };
7767
7768 let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
7769 .expect("missing tiered target should be evaluated");
7770
7771 assert!(!should_resume);
7772 }
7773
7774 #[test]
7775 fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
7776 let info = ObjectInfo::default();
7777 let opts = ObjectOptions::default();
7778
7779 assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none());
7780 }
7781
7782 #[test]
7783 fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() {
7784 let info = ObjectInfo {
7785 delete_marker: true,
7786 ..Default::default()
7787 };
7788 let opts = ObjectOptions::default();
7789
7790 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7791 .expect("delete marker should stop latest-object reads");
7792
7793 assert!(crate::error::is_err_object_not_found(&err));
7794 }
7795
7796 #[test]
7797 fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() {
7798 let info = ObjectInfo {
7799 delete_marker: true,
7800 ..Default::default()
7801 };
7802 let opts = ObjectOptions {
7803 version_id: Some("vid-1".to_string()),
7804 ..Default::default()
7805 };
7806
7807 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7808 .expect("delete marker version reads should be rejected");
7809
7810 assert!(matches!(err, Error::MethodNotAllowed));
7811 }
7812
7813 #[test]
7814 fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() {
7815 let info = ObjectInfo {
7816 delete_marker: true,
7817 ..Default::default()
7818 };
7819 let opts = ObjectOptions {
7820 version_id: Some("vid-1".to_string()),
7821 delete_marker: true,
7822 ..Default::default()
7823 };
7824
7825 let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7826 .expect("delete marker lookup should keep not-found semantics");
7827
7828 assert!(crate::error::is_err_object_not_found(&err));
7829 }
7830
7831 #[test]
7832 fn resolve_latest_object_access_returns_live_object_and_pool_idx() {
7833 let info = ObjectInfo::default();
7834 let opts = ObjectOptions::default();
7835
7836 let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap();
7837
7838 assert_eq!(idx, 7);
7839 assert!(!resolved.delete_marker);
7840 }
7841
7842 #[test]
7843 fn resolve_latest_object_access_rejects_delete_marker_without_version_id() {
7844 let info = ObjectInfo {
7845 delete_marker: true,
7846 ..Default::default()
7847 };
7848 let opts = ObjectOptions::default();
7849
7850 let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7851
7852 assert!(crate::error::is_err_object_not_found(&err));
7853 }
7854
7855 #[test]
7856 fn resolve_latest_object_access_rejects_delete_marker_version_read() {
7857 let info = ObjectInfo {
7858 delete_marker: true,
7859 ..Default::default()
7860 };
7861 let opts = ObjectOptions {
7862 version_id: Some("vid-1".to_string()),
7863 ..Default::default()
7864 };
7865
7866 let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7867
7868 assert!(matches!(err, Error::MethodNotAllowed));
7869 }
7870
7871 #[tokio::test]
7872 async fn latest_versioned_delete_creates_a_new_marker_after_a_current_marker() {
7873 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7874 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7875 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7876 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7877 let bucket = "versioned-delete-marker-identity";
7878 let object = "object.bin";
7879 let opts = ObjectOptions {
7880 versioned: true,
7881 ..Default::default()
7882 };
7883
7884 store
7885 .make_bucket(bucket, &MakeBucketOptions::default())
7886 .await
7887 .expect("bucket should be created");
7888
7889 let first = store
7890 .handle_delete_object(bucket, object, opts.clone())
7891 .await
7892 .expect("first versioned delete should create a marker");
7893 let second = store
7894 .handle_delete_object(bucket, object, opts)
7895 .await
7896 .expect("second versioned delete should create a marker");
7897
7898 assert!(first.delete_marker);
7899 assert!(second.delete_marker);
7900 assert_ne!(
7901 first.version_id, second.version_id,
7902 "each latest versioned delete must persist a distinct delete marker"
7903 );
7904
7905 let versions = store.pools[0]
7906 .get_disks_by_key(object)
7907 .load_file_info_versions_exact(bucket, object)
7908 .await
7909 .expect("delete-marker metadata should decode")
7910 .expect("delete-marker metadata should be persisted");
7911 let marker_ids = versions
7912 .versions
7913 .iter()
7914 .filter(|version| version.deleted)
7915 .filter_map(|version| version.version_id)
7916 .collect::<std::collections::HashSet<_>>();
7917 assert!(marker_ids.contains(&first.version_id.expect("first marker should have a version id")));
7918 assert!(marker_ids.contains(&second.version_id.expect("second marker should have a version id")));
7919 assert_eq!(marker_ids.len(), 2, "both delete markers must remain durable versions");
7920 }
7921
7922 #[tokio::test]
7923 async fn versioned_delete_quorum_failure_rolls_back_and_retries_require_quorum() {
7924 for marker_copies in [0, 2, 4] {
7925 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7926 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7927 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7928 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7929 let bucket = "versioned-delete-quorum-retry";
7930 let object = "object.bin";
7931 let opts = ObjectOptions {
7932 versioned: true,
7933 ..Default::default()
7934 };
7935 let put_opts = ObjectOptions {
7936 no_lock: true,
7937 versioned: true,
7938 ..Default::default()
7939 };
7940 store
7941 .make_bucket(bucket, &MakeBucketOptions::default())
7942 .await
7943 .expect("bucket should be created");
7944 let set = store.pools[0].get_disks_by_key(object);
7945 let payload = vec![0x5a; 4096];
7946 let mut reader = PutObjReader::from_vec(payload.clone());
7947 let original = set
7948 .put_object(bucket, object, &mut reader, &put_opts)
7949 .await
7950 .expect("the versioned 4 KiB object should be written");
7951 let disks = set.disks.read().await.clone();
7952 let marker = rustfs_filemeta::FileInfo {
7954 name: object.to_string(),
7955 version_id: Some(Uuid::new_v4()),
7956 deleted: true,
7957 mark_deleted: true,
7958 mod_time: Some(time::OffsetDateTime::now_utc()),
7959 ..Default::default()
7960 };
7961 for disk in disks.iter().take(marker_copies).flatten() {
7962 disk.delete_version(bucket, object, marker.clone(), true, crate::disk::DeleteOptions::default())
7963 .await
7964 .expect("the initial marker should be persisted on the selected disks");
7965 }
7966 let mut baseline = Vec::new();
7967 for disk in disks.iter().flatten() {
7968 baseline.push(
7969 tokio::fs::read(disk.path().join(bucket).join(object).join("xl.meta"))
7970 .await
7971 .expect("initial metadata should exist"),
7972 );
7973 }
7974 {
7975 let mut online = set.disks.write().await;
7976 online[2] = None;
7977 online[3] = None;
7978 }
7979 for attempt in 1..=3 {
7980 let err = store
7981 .handle_delete_object(bucket, object, opts.clone())
7982 .await
7983 .expect_err("every explicit retry must fail with only two of four disks available");
7984 assert!(
7985 matches!(
7986 &err,
7987 StorageError::InsufficientWriteQuorum(error_bucket, error_object)
7988 if error_bucket == bucket && error_object == object
7989 ),
7990 "expected write quorum failure with {marker_copies} marker copies, attempt {attempt}: {err:?}"
7991 );
7992 for (index, disk) in disks.iter().enumerate() {
7993 let metadata_path = disk
7994 .as_ref()
7995 .expect("saved disk should exist")
7996 .path()
7997 .join(bucket)
7998 .join(object);
7999 let actual = tokio::fs::read(metadata_path.join("xl.meta"))
8000 .await
8001 .expect("rollback must preserve readable metadata");
8002 assert_eq!(
8003 actual, baseline[index],
8004 "failed DELETE must restore exact metadata on disk {index}, attempt {attempt}"
8005 );
8006 let mut entries = tokio::fs::read_dir(&metadata_path)
8007 .await
8008 .expect("the object directory should remain readable");
8009 while let Some(entry) = entries.next_entry().await.expect("directory entries should be readable") {
8010 assert!(
8011 entry.file_name() != "xl.meta.bkp" && !entry.path().join("xl.meta.bkp").exists(),
8012 "rollback must consume staged metadata backups on disk {index}"
8013 );
8014 }
8015 }
8016 }
8017 *set.disks.write().await = disks;
8018 if marker_copies != 0 {
8019 continue;
8020 }
8021 let committed = store
8022 .handle_delete_object(bucket, object, opts)
8023 .await
8024 .expect("DELETE should succeed after write quorum recovers");
8025 assert!(committed.delete_marker);
8026 assert_ne!(committed.version_id, marker.version_id);
8027 let mut restored = Vec::new();
8028 set.get_object_reader(
8029 bucket,
8030 object,
8031 None,
8032 HeaderMap::new(),
8033 &ObjectOptions {
8034 version_id: original.version_id.map(|id| id.to_string()),
8035 ..Default::default()
8036 },
8037 )
8038 .await
8039 .expect("the original version must remain readable")
8040 .stream
8041 .read_to_end(&mut restored)
8042 .await
8043 .expect("the original version should stream");
8044 assert_eq!(restored, payload);
8045 }
8046 }
8047
8048 #[test]
8049 fn latest_versioned_delete_marker_creation_excludes_specialized_deletes() {
8050 assert!(latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8051 versioned: true,
8052 ..Default::default()
8053 }));
8054 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8055 versioned: true,
8056 version_id: Some(Uuid::new_v4().to_string()),
8057 ..Default::default()
8058 }));
8059 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8060 versioned: true,
8061 delete_marker: true,
8062 ..Default::default()
8063 }));
8064 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8065 versioned: true,
8066 data_movement: true,
8067 ..Default::default()
8068 }));
8069 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8070 versioned: true,
8071 replication_request: true,
8072 ..Default::default()
8073 }));
8074 assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8075 versioned: true,
8076 version_suspended: true,
8077 ..Default::default()
8078 }));
8079 }
8080
8081 #[test]
8082 fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
8083 let opts = ObjectOptions {
8084 versioned: true,
8085 ..Default::default()
8086 };
8087
8088 assert!(should_create_delete_marker_for_missing_object(&opts));
8089 }
8090
8091 #[test]
8092 fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
8093 let version_delete = ObjectOptions {
8094 versioned: true,
8095 version_id: Some("vid-1".to_string()),
8096 ..Default::default()
8097 };
8098 let delete_marker_replication = ObjectOptions {
8099 versioned: true,
8100 delete_marker: true,
8101 ..Default::default()
8102 };
8103 let data_movement = ObjectOptions {
8104 versioned: true,
8105 data_movement: true,
8106 ..Default::default()
8107 };
8108
8109 assert!(!should_create_delete_marker_for_missing_object(&version_delete));
8110 assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
8111 assert!(!should_create_delete_marker_for_missing_object(&data_movement));
8112 }
8113
8114 #[test]
8119 fn should_purge_orphan_dir_on_version_not_found_for_dir_key() {
8120 assert!(
8121 should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "ghost/"),
8122 "the real HTTP delete path yields version-not-found on dir keys and must reach the purge"
8123 );
8124 assert!(
8125 should_purge_orphan_dir_on_missing(
8126 &StorageError::VersionNotFound("bucket".into(), "ghost/".into(), Uuid::nil().to_string()),
8127 "ghost/"
8128 ),
8129 "typed VersionNotFound on a dir key must also reach the purge"
8130 );
8131 }
8132
8133 #[test]
8134 fn should_purge_orphan_dir_on_object_not_found_for_dir_key() {
8135 assert!(should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "ghost/"));
8136 assert!(should_purge_orphan_dir_on_missing(
8137 &StorageError::ObjectNotFound("bucket".into(), "ghost/".into()),
8138 "ghost/"
8139 ));
8140 }
8141
8142 #[test]
8143 fn should_not_purge_orphan_dir_for_regular_key_or_other_errors() {
8144 assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "regular.txt"));
8146 assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "regular.txt"));
8147 assert!(!should_purge_orphan_dir_on_missing(&StorageError::ErasureReadQuorum, "ghost/"));
8149 }
8150
8151 #[test]
8152 fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
8153 let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();
8154
8155 assert_eq!(idx, 3);
8156 }
8157
8158 #[test]
8159 fn resolve_decommission_target_pool_idx_result_wraps_error_context() {
8160 let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object")
8161 .expect_err("expected contextual error");
8162 let rendered = err.to_string();
8163
8164 assert!(rendered.contains("failed to select decommission target pool"), "{rendered}");
8165 assert!(rendered.contains("bucket"), "{rendered}");
8166 assert!(rendered.contains("object"), "{rendered}");
8167 assert!(rendered.contains("boom"), "{rendered}");
8168 }
8169
8170 #[test]
8171 fn resolve_decommission_tiered_object_result_passthrough_ok() {
8172 ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object")
8173 .expect("successful decommission result should pass through");
8174 }
8175
8176 #[test]
8177 fn resolve_decommission_tiered_object_result_wraps_error_context() {
8178 let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object")
8179 .expect_err("expected contextual error");
8180 let rendered = err.to_string();
8181
8182 assert!(rendered.contains("failed to decommission tiered object"), "{rendered}");
8183 assert!(rendered.contains("bucket"), "{rendered}");
8184 assert!(rendered.contains("object"), "{rendered}");
8185 assert!(rendered.contains("boom"), "{rendered}");
8186 }
8187
8188 #[test]
8189 fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() {
8190 let err = ECStore::resolve_decommission_tiered_object_result(
8191 Err(Error::DecommissionCapacityBlocked {
8192 message: "target gate busy".to_string(),
8193 }),
8194 "bucket",
8195 "object",
8196 )
8197 .expect_err("expected contextual error");
8198
8199 assert!(matches!(
8200 crate::data_movement::data_movement_stage_source(&err),
8201 Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy"
8202 ));
8203 }
8204
8205 #[test]
8206 fn version_aware_lookup_opts_enables_version_aware_lookup() {
8207 let opts = ObjectOptions {
8208 version_id: Some("vid-1".to_string()),
8209 ..Default::default()
8210 };
8211
8212 let lookup_opts = version_aware_lookup_opts(&opts, true);
8213
8214 assert!(lookup_opts.no_lock);
8215 assert!(lookup_opts.metadata_chg);
8216 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8217 }
8218
8219 #[test]
8220 fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() {
8221 let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true);
8222
8223 assert!(lookup_opts.no_lock);
8224 assert!(!lookup_opts.metadata_chg);
8225 assert!(lookup_opts.version_id.is_none());
8226 }
8227
8228 #[test]
8229 fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() {
8230 let opts = ObjectOptions {
8231 version_id: Some("vid-1".to_string()),
8232 ..Default::default()
8233 };
8234
8235 let lookup_opts = data_movement_pool_lookup_opts(&opts, false);
8236
8237 assert!(!lookup_opts.no_lock);
8238 assert!(lookup_opts.metadata_chg);
8239 assert!(lookup_opts.skip_decommissioned);
8240 assert!(lookup_opts.skip_rebalancing);
8241 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8242 }
8243
8244 #[test]
8245 fn writer_pool_lookup_opts_skips_rebalance_sources() {
8246 let lookup_opts = writer_pool_lookup_opts(
8247 &ObjectOptions {
8248 version_id: Some("vid-1".to_string()),
8249 ..Default::default()
8250 },
8251 true,
8252 );
8253
8254 assert!(lookup_opts.no_lock);
8255 assert!(lookup_opts.metadata_chg);
8256 assert!(lookup_opts.skip_decommissioned);
8257 assert!(lookup_opts.skip_rebalancing);
8258 assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8259 }
8260
8261 #[test]
8262 fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() {
8263 let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true);
8264
8265 assert!(lookup_opts.no_lock);
8266 assert!(!lookup_opts.skip_decommissioned);
8267 assert!(lookup_opts.skip_rebalancing);
8268
8269 let explicit_version = delete_pool_lookup_opts(
8270 &ObjectOptions {
8271 versioned: true,
8272 version_id: Some(uuid::Uuid::new_v4().to_string()),
8273 ..Default::default()
8274 },
8275 true,
8276 );
8277 assert!(!explicit_version.skip_decommissioned);
8278 }
8279
8280 #[test]
8281 fn delete_fans_out_for_unversioned_and_explicit_version_mutations() {
8282 assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1));
8283 assert!(should_delete_from_all_pools(
8284 &ObjectOptions {
8285 versioned: true,
8286 version_id: Some(uuid::Uuid::new_v4().to_string()),
8287 ..Default::default()
8288 },
8289 2,
8290 ));
8291 assert!(!should_delete_from_all_pools(
8292 &ObjectOptions {
8293 versioned: true,
8294 ..Default::default()
8295 },
8296 1,
8297 ));
8298 assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0));
8299 }
8300
8301 #[test]
8302 fn batch_delete_identifies_only_latest_versioned_markers() {
8303 let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test(
8304 s3s::dto::VersioningConfiguration {
8305 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8306 ..Default::default()
8307 },
8308 None,
8309 );
8310 let latest = ObjectToDelete {
8311 object_name: "latest".to_string(),
8312 ..Default::default()
8313 };
8314 assert!(batch_delete_creates_latest_marker(&latest, &versioned));
8315 assert!(!batch_delete_targets_pool(true, Some(1), 0));
8316 assert!(batch_delete_targets_pool(true, Some(1), 1));
8317 assert!(!batch_delete_targets_pool(true, Some(1), 2));
8318
8319 let explicit = ObjectToDelete {
8320 object_name: "explicit".to_string(),
8321 version_id: Some(uuid::Uuid::new_v4()),
8322 ..Default::default()
8323 };
8324 assert!(!batch_delete_creates_latest_marker(&explicit, &versioned));
8325 assert!(batch_delete_targets_pool(false, Some(1), 0));
8326
8327 let unversioned = DeleteReplicationConfigSnapshot::default();
8328 assert!(!batch_delete_creates_latest_marker(&latest, &unversioned));
8329 assert!(batch_delete_targets_pool(false, None, 0));
8330 }
8331
8332 #[test]
8333 fn batch_delete_pool_failures_override_success_in_any_pool_order() {
8334 let success = DeletedObject {
8335 object_name: "object".to_string(),
8336 found: true,
8337 ..Default::default()
8338 };
8339 let source_errors = [
8340 StorageError::ErasureWriteQuorum,
8341 StorageError::NamespaceLockQuorumUnavailable {
8342 mode: "delete_objects_commit",
8343 bucket: "bucket".to_string(),
8344 object: "object".to_string(),
8345 required: 1,
8346 achieved: 0,
8347 },
8348 ];
8349
8350 for source_error in source_errors {
8351 for source_first in [true, false] {
8352 let failed = (DeletedObject::default(), Some(source_error.clone()));
8353 let succeeded = (success.clone(), None);
8354 let pool_results = if source_first {
8355 vec![failed, succeeded]
8356 } else {
8357 vec![succeeded, failed]
8358 };
8359
8360 let (_, error, attempted) =
8361 resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8362
8363 assert!(attempted);
8364 assert_eq!(error, Some(source_error.clone()));
8365 }
8366 }
8367 }
8368
8369 #[test]
8370 fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() {
8371 let success = DeletedObject {
8372 object_name: "object".to_string(),
8373 found: true,
8374 ..Default::default()
8375 };
8376 let missing_errors = [
8377 StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
8378 StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
8379 ];
8380
8381 for missing_error in missing_errors {
8382 let missing = (DeletedObject::default(), Some(missing_error.clone()));
8383 for missing_first in [true, false] {
8384 let succeeded = (success.clone(), None);
8385 let pool_results = if missing_first {
8386 vec![missing.clone(), succeeded]
8387 } else {
8388 vec![succeeded, missing.clone()]
8389 };
8390 let (deleted, error, attempted) =
8391 resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8392
8393 assert!(attempted);
8394 let deleted = deleted.expect("successful pool result should be retained");
8395 assert!(deleted.found);
8396 assert_eq!(deleted.object_name, success.object_name.as_str());
8397 assert!(error.is_none());
8398 }
8399
8400 let missing_only = [missing];
8401 let (_, error, attempted) =
8402 resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error)));
8403 assert!(attempted);
8404 assert_eq!(error, Some(missing_error));
8405 }
8406
8407 let silent_missing = [(DeletedObject::default(), None)];
8408 let (_, error, attempted) =
8409 resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error)));
8410 assert!(attempted);
8411 assert!(error.is_none());
8412 }
8413
8414 #[tokio::test]
8415 async fn lifecycle_null_version_cas_distinguishes_absent_and_replacement_pools() {
8416 use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
8417
8418 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8419 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8420 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8421 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
8422 let bucket = RUSTFS_META_BUCKET;
8423 let suspended_snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
8424 VersioningConfiguration {
8425 status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
8426 ..Default::default()
8427 },
8428 None,
8429 ));
8430 let write_opts = ObjectOptions {
8431 no_lock: true,
8432 version_suspended: true,
8433 ..Default::default()
8434 };
8435 let delete_opts = ObjectOptions {
8436 version_suspended: true,
8437 delete_replication_config_snapshot: Some(Arc::clone(&suspended_snapshot)),
8438 ..Default::default()
8439 };
8440
8441 let success_object = "lifecycle-null-only-in-first-pool";
8442 let mut success_reader = PutObjReader::from_vec(b"single-pool null generation".to_vec());
8443 let success_source = first_set
8444 .put_object(bucket, success_object, &mut success_reader, &write_opts)
8445 .await
8446 .expect("the historical null generation should be written to one pool");
8447 let success_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&success_source)
8448 .expect("the null generation should produce an exact lifecycle target");
8449
8450 let (deleted, errors) = store
8451 .handle_delete_objects(bucket, vec![success_target.clone()], delete_opts.clone())
8452 .await;
8453 assert!(
8454 errors.iter().all(Option::is_none),
8455 "an absent peer pool must not override success: {errors:?}"
8456 );
8457 assert!(deleted[0].found, "the pool containing the exact null generation must report success");
8458
8459 let (deleted, errors) = store
8460 .handle_delete_objects(bucket, vec![success_target], delete_opts.clone())
8461 .await;
8462 assert!(
8463 errors.iter().all(Option::is_none),
8464 "an exact lifecycle delete retry must be idempotent: {errors:?}"
8465 );
8466 assert!(!deleted[0].found, "the retried generation should already be absent from every pool");
8467
8468 let mismatch_object = "lifecycle-null-replaced-in-second-pool";
8469 let mut original_reader = PutObjReader::from_vec(b"original null generation".to_vec());
8470 let original = first_set
8471 .put_object(bucket, mismatch_object, &mut original_reader, &write_opts)
8472 .await
8473 .expect("the original null generation should be written to the first pool");
8474 let stale_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&original)
8475 .expect("the original null generation should produce an exact lifecycle target");
8476 let mut replacement_reader = PutObjReader::from_vec(b"replacement null generation".to_vec());
8477 let replacement = second_set
8478 .put_object(bucket, mismatch_object, &mut replacement_reader, &write_opts)
8479 .await
8480 .expect("the replacement null generation should be written to the second pool");
8481 assert_ne!(original.data_dir, replacement.data_dir);
8482
8483 let (_deleted, errors) = store.handle_delete_objects(bucket, vec![stale_target], delete_opts).await;
8484 assert!(
8485 matches!(errors.as_slice(), [Some(StorageError::PreconditionFailed)]),
8486 "a different null generation in any pool must fail the aggregate CAS: {errors:?}"
8487 );
8488 let retained = second_set
8489 .get_object_info(
8490 bucket,
8491 mismatch_object,
8492 &ObjectOptions {
8493 version_id: Some(Uuid::nil().to_string()),
8494 version_suspended: true,
8495 no_lock: true,
8496 ..Default::default()
8497 },
8498 )
8499 .await
8500 .expect("the replacement null generation must remain after a stale CAS");
8501 assert_eq!(retained.data_dir, replacement.data_dir);
8502 }
8503
8504 #[test]
8505 fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
8506 let lookup_opts = data_movement_pool_lookup_opts(
8507 &ObjectOptions {
8508 version_id: Some("vid-1".to_string()),
8509 ..Default::default()
8510 },
8511 true,
8512 );
8513
8514 assert!(lookup_opts.no_lock);
8515 assert!(lookup_opts.metadata_chg);
8516 assert!(lookup_opts.skip_decommissioned);
8517 assert!(lookup_opts.skip_rebalancing);
8518 }
8519
8520 #[test]
8521 fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() {
8522 let mut opts = ObjectOptions {
8523 data_movement: true,
8524 ..Default::default()
8525 };
8526 assert!(!uses_data_movement_pool_selection(&opts));
8527
8528 DecommissionCapacityOwner {
8529 source_pool_index: 1,
8530 operation_id: Uuid::new_v4(),
8531 generation: 2,
8532 owner_nonce: Uuid::new_v4(),
8533 mutation_id: None,
8534 }
8535 .apply_to(&mut opts);
8536
8537 assert!(uses_data_movement_pool_selection(&opts));
8538 }
8539
8540 #[test]
8541 fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
8542 let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8543 no_lock: false,
8544 skip_decommissioned: false,
8545 ..Default::default()
8546 });
8547
8548 assert!(lookup_opts.skip_decommissioned);
8549 assert!(lookup_opts.skip_rebalancing);
8550 assert!(!lookup_opts.no_lock);
8551 }
8552
8553 #[test]
8554 fn transition_restore_pool_opts_rejects_ambient_no_lock() {
8555 let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8556 no_lock: true,
8557 ..Default::default()
8558 });
8559
8560 assert!(lookup_opts.skip_decommissioned);
8561 assert!(lookup_opts.skip_rebalancing);
8562 assert!(!lookup_opts.no_lock);
8563 }
8564
8565 #[test]
8566 fn delete_objects_lock_names_are_sorted_and_unique() {
8567 let objects = vec![
8568 ObjectToDelete {
8569 object_name: "beta".to_string(),
8570 ..Default::default()
8571 },
8572 ObjectToDelete {
8573 object_name: "alpha".to_string(),
8574 ..Default::default()
8575 },
8576 ObjectToDelete {
8577 object_name: "beta".to_string(),
8578 ..Default::default()
8579 },
8580 ];
8581
8582 assert_eq!(sorted_unique_delete_object_names(&objects), vec!["alpha", "beta"]);
8583 }
8584
8585 async fn new_read_lock_test_store() -> ECStore {
8586 let format = FormatV3::new(1, 2);
8587 let endpoints = vec![
8588 Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
8589 Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
8590 ];
8591 let pool_endpoints = PoolEndpoints {
8592 legacy: false,
8593 set_count: 1,
8594 drives_per_set: 2,
8595 endpoints: Endpoints::from(endpoints),
8596 cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
8597 platform: "test".to_string(),
8598 };
8599 let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
8600 let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
8601 .await
8602 .expect("test sets should be created with empty disks");
8603
8604 ECStore {
8605 id: Uuid::new_v4(),
8606 disk_map: HashMap::new(),
8607 pools: vec![sets],
8608 peer_sys: S3PeerSys::new(&endpoint_pools),
8609 pool_meta: RwLock::new(PoolMeta::default()),
8610 rebalance_meta: RwLock::new(None),
8611 decommission_cancelers: RwLock::new(Vec::new()),
8612 start_gate: Mutex::new(()),
8613 pool_meta_save_gate: Mutex::default(),
8614 ctx: crate::runtime::instance::bootstrap_ctx(),
8615 bucket_fence_registry: std::sync::Arc::default(),
8616 }
8617 }
8618
8619 #[tokio::test]
8620 async fn delete_object_handler_futures_remain_stack_bounded() {
8621 let store = new_read_lock_test_store().await;
8622 let unified_future = store.handle_delete_object_with_journal("bucket", "object", ObjectOptions::default(), None);
8623 let unified_future_size = std::mem::size_of_val(&unified_future);
8624
8625 assert!(
8626 unified_future_size <= 4 * 1024,
8627 "unified delete handler future must remain stack-bounded; measured {unified_future_size} bytes"
8628 );
8629 drop(unified_future);
8630
8631 let batch_future =
8632 store.handle_delete_objects_with_journal_and_accounting("bucket", Vec::new(), ObjectOptions::default(), None);
8633 let batch_future_size = std::mem::size_of_val(&batch_future);
8634 assert!(
8635 batch_future_size <= 4 * 1024,
8636 "batch delete handler future must remain stack-bounded; measured {batch_future_size} bytes"
8637 );
8638 drop(batch_future);
8639
8640 let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default());
8641 let outer_future_size = std::mem::size_of_val(&outer_future);
8642 assert!(
8643 outer_future_size <= 16 * 1024,
8644 "outer delete handler future must remain stack-bounded; measured {outer_future_size} bytes"
8645 );
8646 }
8647
8648 async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
8649 new_prepared_reader_test_store_with_ctx(set_disks, crate::runtime::instance::bootstrap_ctx()).await
8650 }
8651
8652 async fn new_prepared_reader_test_store_with_ctx(
8653 set_disks: &[Arc<SetDisks>],
8654 ctx: Arc<crate::runtime::instance::InstanceContext>,
8655 ) -> ECStore {
8656 let mut pool_configs = Vec::with_capacity(set_disks.len());
8657 let mut pools = Vec::with_capacity(set_disks.len());
8658
8659 for (pool_idx, set_disks) in set_disks.iter().enumerate() {
8660 let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
8661 for endpoint in endpoints.as_mut() {
8662 endpoint.set_pool_index(pool_idx);
8663 }
8664 let pool_config = PoolEndpoints {
8665 legacy: false,
8666 set_count: 1,
8667 drives_per_set: set_disks.set_drive_count,
8668 endpoints,
8669 cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
8670 platform: "test".to_string(),
8671 };
8672 let disks = set_disks.disks.read().await.clone();
8673 let pool = Sets::new_with_instance_ctx(
8674 disks,
8675 &pool_config,
8676 &set_disks.format,
8677 pool_idx,
8678 set_disks.default_parity_count,
8679 Arc::clone(&ctx),
8680 )
8681 .await
8682 .expect("prepared-reader test pool should be created from local disks");
8683 pool_configs.push(pool_config);
8684 pools.push(pool);
8685 }
8686
8687 new_prepared_reader_test_store_from_pools(pools, pool_configs, ctx)
8688 }
8689
8690 fn new_prepared_reader_test_store_from_pools(
8691 pools: Vec<Arc<Sets>>,
8692 pool_configs: Vec<PoolEndpoints>,
8693 ctx: Arc<crate::runtime::instance::InstanceContext>,
8694 ) -> ECStore {
8695 let endpoint_pools = EndpointServerPools::from(pool_configs);
8696 ECStore {
8697 id: Uuid::new_v4(),
8698 disk_map: HashMap::new(),
8699 pools,
8700 peer_sys: S3PeerSys::new_with_instance_ctx(&endpoint_pools, Arc::clone(&ctx)),
8701 pool_meta: RwLock::new(PoolMeta::default()),
8702 rebalance_meta: RwLock::new(None),
8703 decommission_cancelers: RwLock::new(Vec::new()),
8704 start_gate: Mutex::new(()),
8705 pool_meta_save_gate: Mutex::default(),
8706 ctx,
8707 bucket_fence_registry: std::sync::Arc::default(),
8708 }
8709 }
8710
8711 #[tokio::test]
8712 #[serial_test::serial]
8713 async fn object_lock_diagnostics_report_historical_get_blocked_by_put_commit() {
8714 use crate::diagnostics::object_lock::tests::CapturedLockEvents;
8715 use crate::object_api::WriteCompletion;
8716 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
8717 use tracing::{Instrument, instrument::WithSubscriber};
8718
8719 let capture = CapturedLockEvents::default();
8720 temp_env::async_with_vars(
8721 [
8722 (rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true")),
8723 (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
8724 ],
8725 async {
8726 let bucket = "historical-get-lock-diagnostics";
8727 let object = "object.bin";
8728 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8729 let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, ctx.clone()).await;
8730 let store = new_prepared_reader_test_store_with_ctx(&[original], ctx).await;
8731 let set = store.pools[0].get_disks_by_key(object);
8732 set.make_bucket(bucket, &MakeBucketOptions::default())
8733 .await
8734 .expect("create bucket");
8735 let opts = ObjectOptions {
8736 versioned: true,
8737 write_completion: WriteCompletion::TailDrained,
8738 ..Default::default()
8739 };
8740 let mut versions = Vec::new();
8741 for value in [0x11, 0x22, 0x33] {
8742 let info = set
8743 .put_object(bucket, object, &mut PutObjReader::from_vec(vec![value; 8193]), &opts)
8744 .await
8745 .expect("seed version");
8746 versions.push((info.version_id.expect("explicit seed version"), value));
8747 }
8748
8749 let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
8750 let put_set = set.clone();
8751 let put = tokio::spawn(
8752 async move {
8753 put_set
8754 .put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x44; 8193]), &opts)
8755 .await
8756 }
8757 .instrument(tracing::info_span!("request", request_id = "diagnostic-put"))
8758 .with_current_subscriber(),
8759 );
8760 barrier.wait_until_paused().await;
8761
8762 let historical_version = versions[0].0.to_string();
8763 let result = store
8764 .prepare_get_object_reader(
8765 bucket,
8766 object,
8767 None,
8768 HeaderMap::new(),
8769 &ObjectOptions {
8770 version_id: Some(historical_version.clone()),
8771 versioned: true,
8772 ..Default::default()
8773 },
8774 )
8775 .instrument(tracing::info_span!("request", request_id = "diagnostic-get"))
8776 .await;
8777 assert!(
8778 matches!(result, Err(StorageError::Lock(rustfs_lock::LockError::Timeout { .. }))),
8779 "a held PUT commit must preserve the typed GET lock timeout"
8780 );
8781 let rows = capture.rows();
8782 let failure = rows
8783 .iter()
8784 .find(|row| row["fields"]["op"] == "prepare_get_object" && row["fields"]["state"] == "failed")
8785 .expect("failed GET acquisition must be observable before error propagation");
8786 assert_eq!(failure["fields"]["requested_version_id"], historical_version);
8787 assert_eq!(failure["fields"]["failure"], "timeout");
8788 assert_eq!(failure["fields"]["timeout_ms"], 1000);
8789 assert!(failure["fields"]["acquire_ms"].as_u64().expect("wait duration") >= 1000);
8790 assert!(
8791 failure["spans"]
8792 .as_array()
8793 .expect("request spans")
8794 .iter()
8795 .any(|span| span["request_id"] == "diagnostic-get")
8796 );
8797 let put_acquired = rows
8798 .iter()
8799 .rev()
8800 .find(|row| row["fields"]["op"] == "put_object_commit" && row["fields"]["state"] == "acquired")
8801 .expect("PUT lock holder event");
8802 assert_ne!(put_acquired["fields"]["lock_attempt_id"], failure["fields"]["lock_attempt_id"]);
8803 assert!(
8804 !rows
8805 .iter()
8806 .any(|row| row["fields"]["lock_attempt_id"] == put_acquired["fields"]["lock_attempt_id"]
8807 && row["fields"]["state"] == "guard_dropped"),
8808 "a paused commit still owns the lock"
8809 );
8810
8811 barrier.release();
8812 let new_version = put
8813 .await
8814 .expect("PUT task")
8815 .expect("new version commits")
8816 .version_id
8817 .expect("new version id");
8818 versions.push((new_version, 0x44));
8819 for (version_id, value) in versions {
8820 let prepared = store
8821 .prepare_get_object_reader(
8822 bucket,
8823 object,
8824 None,
8825 HeaderMap::new(),
8826 &ObjectOptions {
8827 version_id: Some(version_id.to_string()),
8828 versioned: true,
8829 ..Default::default()
8830 },
8831 )
8832 .await
8833 .expect("every acknowledged version remains readable");
8834 assert_eq!(prepared.object_info().version_id, Some(version_id));
8835 let mut reader = prepared.into_reader().await.expect("open historical body");
8836 let mut body = Vec::new();
8837 reader
8838 .stream
8839 .read_to_end(&mut body)
8840 .await
8841 .expect("read complete version body");
8842 assert_eq!(body, vec![value; 8193]);
8843 }
8844 }
8845 .with_subscriber(capture.subscriber()),
8846 )
8847 .await;
8848 }
8849
8850 async fn multipool_version_test_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
8851 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8852 let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8853 let (second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8854 dirs.extend(second_dirs);
8855 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first_set, second_set], ctx).await);
8856 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
8857 store
8858 .handle_make_bucket(
8859 bucket,
8860 &MakeBucketOptions {
8861 versioning_enabled: true,
8862 ..Default::default()
8863 },
8864 )
8865 .await
8866 .expect("create the versioned bucket in both pools");
8867 (dirs, store)
8868 }
8869
8870 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8871 async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
8872 let bucket = "retired-marker-c11";
8873 let object = "marker.bin";
8874 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8875 let (dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
8876 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
8877 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8878 let create = MakeBucketOptions {
8879 versioning_enabled: true,
8880 ..Default::default()
8881 };
8882 store.handle_make_bucket(bucket, &create).await.expect("create old bucket");
8883 let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old identity");
8884 let mut historical = Vec::new();
8885 for value in 1..=3 {
8886 historical.push(
8887 store
8888 .put_object(
8889 bucket,
8890 "history.bin",
8891 &mut PutObjReader::from_vec(vec![value; 4097]),
8892 &ObjectOptions {
8893 versioned: true,
8894 ..Default::default()
8895 },
8896 )
8897 .await
8898 .expect("write old version")
8899 .version_id
8900 .unwrap(),
8901 );
8902 }
8903 let marker = store
8904 .delete_object(
8905 bucket,
8906 object,
8907 ObjectOptions {
8908 versioned: true,
8909 ..Default::default()
8910 },
8911 )
8912 .await
8913 .expect("create old marker")
8914 .version_id
8915 .expect("explicit marker version");
8916 assert!(
8917 store
8918 .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8919 .await
8920 .is_err()
8921 );
8922 assert!(
8923 !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8924 .await
8925 .unwrap(),
8926 "failed bucket deletion cannot publish retirement"
8927 );
8928 let set = &store.pools[0].disk_set[0];
8929 let before = set.disks.read().await.clone();
8930 for disk in before.iter().flatten() {
8931 let info = disk
8932 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
8933 .await
8934 .expect("read original marker");
8935 assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation));
8936 }
8937 for slot in 12..16 {
8940 set.disks.write().await[slot] = None;
8941 }
8942 for version in historical {
8943 store
8944 .delete_object(
8945 bucket,
8946 "history.bin",
8947 ObjectOptions {
8948 versioned: true,
8949 version_id: Some(version.to_string()),
8950 ..Default::default()
8951 },
8952 )
8953 .await
8954 .expect("delete exact old data version with four drives offline");
8955 }
8956 store
8957 .delete_object(
8958 bucket,
8959 object,
8960 ObjectOptions {
8961 versioned: true,
8962 version_id: Some(marker.to_string()),
8963 ..Default::default()
8964 },
8965 )
8966 .await
8967 .expect("delete exact old marker with four drives offline");
8968 store
8969 .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8970 .await
8971 .expect("delete empty old bucket");
8972 assert!(
8973 crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8974 .await
8975 .expect("durable retirement")
8976 );
8977 store
8978 .handle_make_bucket(bucket, &create)
8979 .await
8980 .expect("recreate same bucket name");
8981 let current = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
8982 assert_ne!(old_incarnation, current);
8983 let mut new_versions = Vec::new();
8984 for (key, payload) in [
8985 (object, b"new generation reused key".as_slice()),
8986 ("new.bin", b"new generation control".as_slice()),
8987 ] {
8988 let info = store
8989 .put_object(
8990 bucket,
8991 key,
8992 &mut PutObjReader::from_vec(payload.to_vec()),
8993 &ObjectOptions {
8994 versioned: true,
8995 ..Default::default()
8996 },
8997 )
8998 .await
8999 .expect("write new generation");
9000 new_versions.push((key, info.version_id.unwrap(), payload));
9001 }
9002 *set.disks.write().await = before.clone();
9003 let heal = rustfs_heal_contracts::heal_channel::HealOpts {
9004 remove: true,
9005 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9006 ..Default::default()
9007 };
9008 let observed = store
9009 .clone()
9010 .inner_list_object_versions(bucket, "", None, None, None, 100)
9011 .await
9012 .expect("list after rejoin");
9013 assert_eq!(observed.objects.len(), 2, "stale marker must not contaminate listing");
9014 assert!(observed.objects.iter().all(|info| !info.delete_marker));
9015 for (key, version, _) in &new_versions {
9016 let repaired = store
9017 .heal_object_with_proof(bucket, key, &version.to_string(), &heal)
9018 .await
9019 .expect("heal new generation");
9020 assert!(repaired.error.is_none(), "new version heal: {:?}", repaired.error);
9021 }
9022 let no_remove = store
9023 .heal_object_with_proof(
9024 bucket,
9025 object,
9026 &marker.to_string(),
9027 &rustfs_heal_contracts::heal_channel::HealOpts { remove: false, ..heal },
9028 )
9029 .await
9030 .expect("inspect marker without removal");
9031 assert!(no_remove.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9032 assert!(no_remove.absence.is_none());
9033 let dry = store
9034 .heal_object_with_proof(
9035 bucket,
9036 object,
9037 &marker.to_string(),
9038 &rustfs_heal_contracts::heal_channel::HealOpts { dry_run: true, ..heal },
9039 )
9040 .await
9041 .expect("dry run");
9042 assert!(dry.absence.is_none());
9043 for disk in before.iter().skip(12).flatten() {
9044 assert!(
9045 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9046 .await
9047 .is_ok()
9048 );
9049 }
9050 let original_marker = before[12]
9051 .as_ref()
9052 .unwrap()
9053 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9054 .await
9055 .unwrap();
9056 for stamp in [None, Some(current), Some(Uuid::new_v4())] {
9057 for disk in before.iter().skip(12).flatten() {
9058 let mut changed = original_marker.clone();
9059 changed.metadata.clear();
9060 if let Some(stamp) = stamp {
9061 changed.set_delete_marker_incarnation(stamp);
9062 }
9063 disk.write_metadata("", bucket, object, changed).await.unwrap();
9064 }
9065 let unresolved = store
9066 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9067 .await
9068 .unwrap();
9069 assert!(
9070 unresolved.error.as_ref().is_some_and(Error::is_retired_marker_deferred),
9071 "legacy/current/unproven markers must be deferred: {:?}",
9072 unresolved.error
9073 );
9074 assert!(unresolved.absence.is_none());
9075 for disk in before.iter().skip(12).flatten() {
9076 assert!(
9077 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9078 .await
9079 .is_ok()
9080 );
9081 disk.write_metadata("", bucket, object, original_marker.clone())
9082 .await
9083 .unwrap();
9084 }
9085 }
9086 set.disks.write().await[15] = None;
9087 let incomplete = store
9088 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9089 .await
9090 .unwrap();
9091 assert!(incomplete.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9092 assert!(incomplete.absence.is_none(), "offline targets cannot be omitted from a cleanup receipt");
9093 set.disks.write().await[15] = before[15].clone();
9094 let failure =
9095 crate::set_disk::DanglingDeleteFailure::install(bucket, object, 15, crate::disk::error::DiskError::FaultyDisk);
9096 let partial = store
9097 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9098 .await
9099 .unwrap();
9100 assert!(partial.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9101 assert!(
9102 partial.absence.is_none(),
9103 "fifteen successful/absent disks cannot hide one remaining marker"
9104 );
9105 assert!(
9106 before[15]
9107 .as_ref()
9108 .unwrap()
9109 .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9110 .await
9111 .is_ok()
9112 );
9113 drop(failure);
9114 let cleaned = store
9115 .heal_object_at_incarnation(bucket, object, &marker.to_string(), current, &heal)
9116 .await
9117 .expect("cleanup retired marker");
9118 assert!(cleaned.error.is_none(), "cleanup error: {:?}", cleaned.error);
9119 let proof = cleaned.absence.expect("complete cleanup receipt");
9120 assert!(proof.removed);
9121 assert_eq!(proof.bucket_incarnation_id, current);
9122 for disk in before.iter().flatten() {
9123 assert!(matches!(
9124 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9125 .await,
9126 Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9127 ));
9128 }
9129 let replay = store
9130 .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9131 .await
9132 .expect("idempotent replay");
9133 assert!(replay.error.is_none());
9134 assert!(!replay.absence.expect("authoritative absence receipt").removed);
9135 for (key, version, payload) in new_versions {
9136 let mut reader = store
9137 .handle_get_object_reader(
9138 bucket,
9139 key,
9140 None,
9141 HeaderMap::new(),
9142 &ObjectOptions {
9143 version_id: Some(version.to_string()),
9144 ..Default::default()
9145 },
9146 )
9147 .await
9148 .expect("new generation still readable");
9149 let mut actual = Vec::new();
9150 reader.stream.read_to_end(&mut actual).await.expect("read new bytes");
9151 assert_eq!(actual, payload);
9152 }
9153 assert_eq!(dirs.len(), 16);
9154 }
9155
9156 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9157 async fn current_marker_delete_ack_removes_stale_rejoined_copy() {
9158 let bucket = "current-marker-stale-rejoin";
9159 let object = "marker.bin";
9160 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
9161 let (_dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
9162 let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
9163 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
9164 store
9165 .handle_make_bucket(
9166 bucket,
9167 &MakeBucketOptions {
9168 versioning_enabled: true,
9169 ..Default::default()
9170 },
9171 )
9172 .await
9173 .expect("create versioned bucket");
9174 let mut history = Vec::new();
9175 for value in 1..=3_u8 {
9176 let payload = vec![value; 4097];
9177 let version = store
9178 .put_object(
9179 bucket,
9180 object,
9181 &mut PutObjReader::from_vec(payload.clone()),
9182 &ObjectOptions {
9183 versioned: true,
9184 ..Default::default()
9185 },
9186 )
9187 .await
9188 .expect("write the historical data versions")
9189 .version_id
9190 .expect("historical version must have an explicit identity");
9191 history.push((version, payload));
9192 }
9193 let marker = store
9194 .delete_object(
9195 bucket,
9196 object,
9197 ObjectOptions {
9198 versioned: true,
9199 ..Default::default()
9200 },
9201 )
9202 .await
9203 .expect("create current marker")
9204 .version_id
9205 .expect("marker has explicit version");
9206 let current = store
9207 .bucket_incarnation_id_from_disk(bucket)
9208 .await
9209 .expect("bucket incarnation");
9210 let set = &store.pools[0].disk_set[0];
9211 let before = set.disks.read().await.clone();
9212 for slot in 12..16 {
9213 set.disks.write().await[slot] = None;
9214 }
9215 store
9216 .delete_object(
9217 bucket,
9218 object,
9219 ObjectOptions {
9220 versioned: true,
9221 version_id: Some(marker.to_string()),
9222 ..Default::default()
9223 },
9224 )
9225 .await
9226 .expect("acknowledged delete should remove the marker from the online quorum");
9227 *set.disks.write().await = before.clone();
9228
9229 for disk in before.iter().take(12).flatten() {
9230 assert!(matches!(
9231 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9232 .await,
9233 Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9234 ));
9235 }
9236 for disk in before.iter().skip(12).flatten() {
9237 assert!(
9238 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9239 .await
9240 .is_ok(),
9241 "offline member must retain the stale current marker"
9242 );
9243 }
9244 let healed = store
9245 .heal_object_at_incarnation(
9246 bucket,
9247 object,
9248 &marker.to_string(),
9249 current,
9250 &rustfs_heal_contracts::heal_channel::HealOpts {
9251 remove: true,
9252 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9253 ..Default::default()
9254 },
9255 )
9256 .await
9257 .expect("stale current marker heal should complete");
9258 assert!(healed.error.is_none(), "stale current marker heal: {:?}", healed.error);
9259 assert!(healed.absence.as_ref().is_some_and(|proof| proof.removed));
9260 for disk in before.iter().flatten() {
9261 assert!(matches!(
9262 disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9263 .await,
9264 Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9265 ));
9266 }
9267 let replay = store
9268 .heal_object_at_incarnation(
9269 bucket,
9270 object,
9271 &marker.to_string(),
9272 current,
9273 &rustfs_heal_contracts::heal_channel::HealOpts {
9274 remove: true,
9275 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
9276 ..Default::default()
9277 },
9278 )
9279 .await
9280 .expect("replay the exact removed marker identity");
9281 assert!(replay.error.is_none(), "exact marker replay: {:?}", replay.error);
9282 let absence = replay.absence.expect("exact replay must prove authoritative absence");
9283 assert!(!absence.removed, "replay must not claim another physical repair");
9284 assert_eq!(absence.bucket_incarnation_id, current);
9285 assert_eq!(absence.version_id, marker.to_string());
9286 let listed = store
9287 .clone()
9288 .inner_list_object_versions(bucket, "", None, None, None, 100)
9289 .await
9290 .expect("list surviving versions after marker cleanup");
9291 assert_eq!(listed.objects.len(), history.len(), "an absent marker is not enumerable in a new walk");
9292 assert!(listed.objects.iter().all(|info| !info.delete_marker));
9293 for (version, expected) in history {
9294 assert!(listed.objects.iter().any(|info| info.version_id == Some(version)));
9295 let mut reader = store
9296 .handle_get_object_reader(
9297 bucket,
9298 object,
9299 None,
9300 HeaderMap::new(),
9301 &ObjectOptions {
9302 version_id: Some(version.to_string()),
9303 ..Default::default()
9304 },
9305 )
9306 .await
9307 .expect("historical data must remain readable after cleanup and replay");
9308 let mut actual = Vec::new();
9309 reader
9310 .stream
9311 .read_to_end(&mut actual)
9312 .await
9313 .expect("read all historical bytes");
9314 assert_eq!(actual, expected);
9315 }
9316 }
9317
9318 #[tokio::test]
9319 async fn multipool_delete_marker_stays_with_existing_versions() {
9320 let bucket = "multipool-marker-routing";
9321 let object = "history.bin";
9322 let (_dirs, store) = multipool_version_test_store(bucket).await;
9323
9324 let mut expected_versions = Vec::new();
9325 for value in 1..=3_u8 {
9326 let written = store.pools[1]
9327 .put_object(
9328 bucket,
9329 object,
9330 &mut PutObjReader::from_vec(vec![value; 4097]),
9331 &ObjectOptions {
9332 versioned: true,
9333 user_defined: HashMap::from([
9334 (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), format!("generation={value}")),
9335 ("x-amz-meta-generation".to_string(), value.to_string()),
9336 ]),
9337 ..Default::default()
9338 },
9339 )
9340 .await
9341 .expect("write a historical version deterministically to pool 1");
9342 expected_versions.push(written.version_id.expect("versioned PUT must acknowledge a UUID"));
9343 }
9344 let marker = store
9345 .delete_object(
9346 bucket,
9347 object,
9348 ObjectOptions {
9349 versioned: true,
9350 ..Default::default()
9351 },
9352 )
9353 .await
9354 .expect("delete the current version");
9355 assert!(marker.delete_marker);
9356 let marker_id = marker.version_id.expect("DELETE must acknowledge a marker UUID");
9357 assert!(!expected_versions.contains(&marker_id));
9358
9359 let local = store.pools[1]
9360 .clone()
9361 .inner_list_object_versions(bucket, object, None, None, None, 10)
9362 .await
9363 .expect("list the object-owning pool");
9364 assert_eq!(local.objects.len(), 4, "the marker must be committed beside the three existing versions");
9365 assert_eq!(local.objects[0].version_id, Some(marker_id));
9366 assert!(local.objects[0].delete_marker && local.objects[0].is_latest);
9367 let versions = store
9368 .clone()
9369 .inner_list_object_versions(bucket, object, None, None, None, 10)
9370 .await
9371 .expect("list all pools");
9372 assert_eq!(versions.objects.len(), 4);
9373 assert_eq!(versions.objects.iter().filter(|version| version.is_latest).count(), 1);
9374 for (index, version_id) in expected_versions.iter().copied().enumerate() {
9375 assert!(
9376 versions
9377 .objects
9378 .iter()
9379 .any(|version| version.version_id == Some(version_id) && !version.is_latest)
9380 );
9381 let mut reader = store
9382 .handle_get_object_reader(
9383 bucket,
9384 object,
9385 None,
9386 HeaderMap::new(),
9387 &ObjectOptions {
9388 version_id: Some(version_id.to_string()),
9389 ..Default::default()
9390 },
9391 )
9392 .await
9393 .expect("historical version should remain readable");
9394 let mut payload = Vec::new();
9395 reader
9396 .stream
9397 .read_to_end(&mut payload)
9398 .await
9399 .expect("read all historical bytes");
9400 let value = u8::try_from(index + 1).expect("fixture generation fits u8");
9401 assert_eq!(payload, vec![value; 4097]);
9402 let listed = versions
9403 .objects
9404 .iter()
9405 .find(|version| version.version_id == Some(version_id))
9406 .expect("listed historical version");
9407 assert_eq!(listed.user_tags.as_str(), format!("generation={value}"));
9408 assert_eq!(listed.user_defined.get("x-amz-meta-generation"), Some(&value.to_string()));
9409 }
9410
9411 let repeated = store
9412 .delete_object(
9413 bucket,
9414 object,
9415 ObjectOptions {
9416 versioned: true,
9417 ..Default::default()
9418 },
9419 )
9420 .await
9421 .expect("repeat simple DELETE");
9422 assert!(repeated.delete_marker);
9423 assert_ne!(
9424 repeated.version_id,
9425 Some(marker_id),
9426 "each enabled-versioning DELETE creates a new marker"
9427 );
9428 store
9429 .delete_object(
9430 bucket,
9431 object,
9432 ObjectOptions {
9433 versioned: true,
9434 version_id: repeated.version_id.map(|id| id.to_string()),
9435 ..Default::default()
9436 },
9437 )
9438 .await
9439 .expect("remove the newest marker by identity");
9440 store
9441 .delete_object(
9442 bucket,
9443 object,
9444 ObjectOptions {
9445 versioned: true,
9446 version_id: Some(marker_id.to_string()),
9447 ..Default::default()
9448 },
9449 )
9450 .await
9451 .expect("remove the original marker by identity");
9452 let current = store
9453 .get_object_info(bucket, object, &ObjectOptions::default())
9454 .await
9455 .expect("previous version becomes current");
9456 assert_eq!(current.version_id, expected_versions.last().copied());
9457 assert!(!current.delete_marker);
9458 }
9459
9460 #[tokio::test]
9461 async fn multipool_existing_split_history_lists_and_paginates_without_metadata_writes() {
9462 for marker_pool in 0..2 {
9463 let bucket = format!("multipool-split-history-{marker_pool}");
9464 let object = "history.bin";
9465 let (dirs, store) = multipool_version_test_store(&bucket).await;
9466 let mut expected = Vec::new();
9467 for value in 1..=3_u8 {
9468 let version = store.pools[1 - marker_pool]
9469 .put_object(
9470 &bucket,
9471 object,
9472 &mut PutObjReader::from_vec(vec![value; 4097]),
9473 &ObjectOptions {
9474 versioned: true,
9475 ..Default::default()
9476 },
9477 )
9478 .await
9479 .expect("seed history through the owning pool's normal write path");
9480 expected.push(version.version_id);
9481 }
9482 let marker = store.pools[marker_pool]
9483 .delete_object(
9484 &bucket,
9485 object,
9486 ObjectOptions {
9487 versioned: true,
9488 ..Default::default()
9489 },
9490 )
9491 .await
9492 .expect("reproduce the previously committed split marker with a normal pool DELETE");
9493 assert!(marker.delete_marker);
9494 expected.push(marker.version_id);
9495 expected.reverse();
9496 let mut before = Vec::new();
9497 for dir in &dirs {
9498 before.push(
9499 tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9500 .await
9501 .expect("snapshot persisted version metadata"),
9502 );
9503 }
9504 for max_keys in [1, 2, 4, 10] {
9505 let mut key_marker = None;
9506 let mut version_marker = None;
9507 let mut listed = Vec::new();
9508 let mut completed = false;
9509 for _ in 0..6 {
9510 let page = store
9511 .clone()
9512 .inner_list_object_versions(&bucket, object, key_marker.clone(), version_marker.clone(), None, max_keys)
9513 .await
9514 .expect("read a complete merged version page");
9515 listed.extend(page.objects);
9516 if !page.is_truncated {
9517 completed = true;
9518 break;
9519 }
9520 assert_ne!(
9521 (&page.next_marker, &page.next_version_idmarker),
9522 (&key_marker, &version_marker),
9523 "version cursor must advance"
9524 );
9525 key_marker = page.next_marker;
9526 version_marker = page.next_version_idmarker;
9527 }
9528 assert!(completed, "pagination must terminate");
9529 assert_eq!(listed.iter().map(|version| version.version_id).collect::<Vec<_>>(), expected);
9530 assert!(listed[0].delete_marker && listed[0].is_latest);
9531 assert!(listed[1..].iter().all(|version| !version.delete_marker && !version.is_latest));
9532 }
9533 let visible = store
9534 .clone()
9535 .list_objects_generic(&bucket, "", None, None, 10, false)
9536 .await
9537 .expect("list current objects");
9538 assert!(visible.objects.is_empty(), "the global current marker hides the object");
9539 for (dir, before) in dirs.iter().zip(before) {
9540 assert_eq!(
9541 tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9542 .await
9543 .expect("read unchanged metadata"),
9544 before
9545 );
9546 }
9547 }
9548 }
9549
9550 #[tokio::test]
9551 async fn multipool_conflicting_version_metadata_fails_the_listing_request() {
9552 let bucket = "multipool-version-conflict";
9553 let (_dirs, store) = multipool_version_test_store(bucket).await;
9554 store.pools[0]
9555 .put_object(
9556 bucket,
9557 "a.bin",
9558 &mut PutObjReader::from_vec(b"preceding result".to_vec()),
9559 &ObjectOptions {
9560 versioned: true,
9561 ..Default::default()
9562 },
9563 )
9564 .await
9565 .expect("seed an entry before the conflict");
9566 let version_id = Uuid::new_v4();
9567 let mod_time = OffsetDateTime::now_utc();
9568 let mut etags = Vec::new();
9569 for (pool_idx, value) in [(0, 1), (1, 2)] {
9570 let written = store.pools[pool_idx]
9571 .put_object(
9572 bucket,
9573 "z.bin",
9574 &mut PutObjReader::from_vec(vec![value; 4097]),
9575 &ObjectOptions {
9576 versioned: true,
9577 version_id: Some(version_id.to_string()),
9578 mod_time: Some(mod_time),
9579 ..Default::default()
9580 },
9581 )
9582 .await
9583 .expect("persist independent conflicting copies");
9584 assert_eq!(written.version_id, Some(version_id));
9585 etags.push(written.etag);
9586 }
9587 assert_ne!(etags[0], etags[1], "fixture must contain a semantic conflict");
9588 let err = store
9589 .clone()
9590 .inner_list_object_versions(bucket, "", None, None, None, 10)
9591 .await
9592 .expect_err("partial output must not hide the merge error");
9593 assert_eq!(err, StorageError::FileCorrupt);
9594
9595 let cancellation = tokio_util::sync::CancellationToken::new();
9596 let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
9597 let walk = store
9598 .clone()
9599 .walk(cancellation.clone(), bucket, "", sender, WalkOptions::default());
9600 let drain = async { while receiver.recv().await.is_some() {} };
9601 let (walk_result, ()) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(walk, drain) })
9602 .await
9603 .expect("bounded walk output must drain and terminate on a merge error");
9604 assert_eq!(
9605 walk_result.expect_err("walk must report the same metadata conflict"),
9606 StorageError::FileCorrupt
9607 );
9608 assert!(!cancellation.is_cancelled(), "worker failure must not cancel the caller's request");
9609 }
9610
9611 #[tokio::test]
9612 async fn multipool_marker_rejects_unwritable_owner_without_falling_back() {
9613 let bucket = "multipool-unwritable-owner";
9614 let object = "history.bin";
9615 let (_dirs, store) = multipool_version_test_store(bucket).await;
9616 store.pools[1]
9617 .put_object(
9618 bucket,
9619 object,
9620 &mut PutObjReader::from_vec(vec![1; 4097]),
9621 &ObjectOptions {
9622 versioned: true,
9623 ..Default::default()
9624 },
9625 )
9626 .await
9627 .expect("seed the nonzero owner");
9628 *store.pool_meta.write().await = PoolMeta {
9629 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
9630 ..Default::default()
9631 };
9632 let err = store
9633 .delete_object(
9634 bucket,
9635 object,
9636 ObjectOptions {
9637 versioned: true,
9638 ..Default::default()
9639 },
9640 )
9641 .await
9642 .expect_err("suspended owner must reject marker creation");
9643 assert_eq!(err, StorageError::SlowDown);
9644 *store.pool_meta.write().await = PoolMeta::default();
9645
9646 let mut rebalancing = crate::services::rebalance::RebalanceStats {
9647 participating: true,
9648 ..Default::default()
9649 };
9650 rebalancing.info.status = crate::services::rebalance::RebalStatus::Started;
9651 *store.rebalance_meta.write().await = Some(crate::services::rebalance::RebalanceMeta {
9652 pool_stats: vec![crate::services::rebalance::RebalanceStats::default(), rebalancing],
9653 ..Default::default()
9654 });
9655 let error = store
9656 .delete_object(
9657 bucket,
9658 object,
9659 ObjectOptions {
9660 versioned: true,
9661 ..Default::default()
9662 },
9663 )
9664 .await;
9665 *store.rebalance_meta.write().await = None;
9666 assert_eq!(
9667 error.expect_err("rebalance must not hide the owner during marker lookup"),
9668 StorageError::SlowDown
9669 );
9670
9671 let quorum_bucket = RUSTFS_META_BUCKET;
9674 store.pools[1]
9675 .put_object(
9676 quorum_bucket,
9677 object,
9678 &mut PutObjReader::from_vec(vec![1; 4097]),
9679 &ObjectOptions {
9680 versioned: true,
9681 ..Default::default()
9682 },
9683 )
9684 .await
9685 .expect("seed the object-quorum fixture");
9686 for pool in &store.pools {
9687 pool.put_object(
9688 quorum_bucket,
9689 "split.bin",
9690 &mut PutObjReader::from_vec(vec![2; 4097]),
9691 &ObjectOptions {
9692 versioned: true,
9693 ..Default::default()
9694 },
9695 )
9696 .await
9697 .expect("seed another key with history in both pools");
9698 }
9699
9700 let owner = &store.pools[1].disk_set[0];
9701 let healthy = owner.disks.read().await.clone();
9702 for disk in owner.disks.write().await.iter_mut().skip(1) {
9703 *disk = None;
9704 }
9705 let error = store
9706 .delete_object(
9707 quorum_bucket,
9708 object,
9709 ObjectOptions {
9710 versioned: true,
9711 ..Default::default()
9712 },
9713 )
9714 .await;
9715 let split_error = store
9716 .delete_object(
9717 quorum_bucket,
9718 "split.bin",
9719 ObjectOptions {
9720 versioned: true,
9721 ..Default::default()
9722 },
9723 )
9724 .await;
9725 *owner.disks.write().await = healthy;
9726 assert_eq!(
9727 error.expect_err("subquorum owner must not become a new-pool marker"),
9728 StorageError::ErasureWriteQuorum
9729 );
9730 assert_eq!(
9731 split_error.expect_err("a readable older pool does not prove the global current version"),
9732 StorageError::ErasureWriteQuorum
9733 );
9734 let split = store.pools[0]
9735 .clone()
9736 .inner_list_object_versions(quorum_bucket, "split.bin", None, None, None, 10)
9737 .await
9738 .expect("inspect the readable older pool");
9739 assert_eq!(split.objects.len(), 1);
9740 assert!(!split.objects[0].delete_marker);
9741 let other = store.pools[0]
9742 .clone()
9743 .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9744 .await
9745 .expect("inspect the other pool");
9746 assert!(other.objects.is_empty());
9747 let history = store.pools[1]
9748 .clone()
9749 .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9750 .await
9751 .expect("inspect the restored owner");
9752 assert_eq!(history.objects.len(), 1);
9753 assert!(!history.objects[0].delete_marker);
9754 }
9755
9756 #[tokio::test]
9757 async fn multipool_suspended_and_batch_markers_keep_the_null_slot_on_the_owner() {
9758 let bucket = "multipool-suspended-markers";
9759 let object = "history.bin";
9760 let (_dirs, store) = multipool_version_test_store(bucket).await;
9761 let original = store.pools[1]
9762 .put_object(
9763 bucket,
9764 object,
9765 &mut PutObjReader::from_vec(vec![1; 4097]),
9766 &ObjectOptions {
9767 versioned: true,
9768 ..Default::default()
9769 },
9770 )
9771 .await
9772 .expect("seed the UUID history in pool 1");
9773 store
9774 .update_bucket_metadata_config(
9775 bucket,
9776 crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
9777 b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
9778 )
9779 .await
9780 .expect("persist suspended bucket versioning");
9781 let null = store
9782 .put_object(
9783 bucket,
9784 object,
9785 &mut PutObjReader::from_vec(vec![2; 4097]),
9786 &ObjectOptions {
9787 version_suspended: true,
9788 ..Default::default()
9789 },
9790 )
9791 .await
9792 .expect("write the null version beside the history");
9793 assert!(null.version_id.is_none_or(|id| id.is_nil()));
9794 for _ in 0..2 {
9795 let marker = store
9796 .delete_object(
9797 bucket,
9798 object,
9799 ObjectOptions {
9800 version_suspended: true,
9801 ..Default::default()
9802 },
9803 )
9804 .await
9805 .expect("replace the null slot with a marker");
9806 assert!(marker.delete_marker);
9807 assert!(marker.version_id.is_none_or(|id| id.is_nil()));
9808 }
9809 let (deleted, errors) = store
9810 .delete_objects(
9811 bucket,
9812 vec![ObjectToDelete {
9813 object_name: object.to_string(),
9814 ..Default::default()
9815 }],
9816 ObjectOptions::default(),
9817 )
9818 .await;
9819 assert!(errors.iter().all(Option::is_none), "batch DELETE should succeed: {errors:?}");
9820 assert_eq!(deleted.len(), 1);
9821 assert!(deleted[0].delete_marker);
9822 let versions = store.pools[1]
9823 .clone()
9824 .inner_list_object_versions(bucket, object, None, None, None, 10)
9825 .await
9826 .expect("inspect the owner after batch DELETE");
9827 assert_eq!(versions.objects.len(), 2, "only one null marker and the UUID history remain");
9828 assert!(versions.objects[0].delete_marker && versions.objects[0].is_latest);
9829 assert_eq!(versions.objects[1].version_id, original.version_id);
9830 let other = store.pools[0]
9831 .clone()
9832 .inner_list_object_versions(bucket, object, None, None, None, 10)
9833 .await
9834 .expect("inspect the unused pool");
9835 assert!(other.objects.is_empty());
9836 }
9837
9838 async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
9839 assert_pool_writer_is_blocked(store, 0, bucket, object).await;
9840 }
9841
9842 async fn assert_pool_writer_is_blocked(store: &ECStore, pool_idx: usize, bucket: &str, object: &str) {
9843 let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9844 let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9845 let err = lock
9846 .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
9847 .await
9848 .expect_err("prepared read lock should block the writer");
9849 assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
9850 }
9851
9852 async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
9853 acquire_pool_writer(store, 0, bucket, object).await
9854 }
9855
9856 async fn acquire_pool_writer(
9857 store: &ECStore,
9858 pool_idx: usize,
9859 bucket: &str,
9860 object: &str,
9861 ) -> rustfs_lock::NamespaceLockGuard {
9862 let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9863 let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9864 lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
9865 .await
9866 .expect("prepared read lock should have been released")
9867 }
9868
9869 #[tokio::test]
9870 #[serial_test::serial(body_cache_hook)]
9871 async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
9872 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9873 let store = new_prepared_reader_test_store(&[set_disks]).await;
9874 let bucket = "prepared-reader-hook-miss";
9875 let object = "object.bin";
9876 let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
9877 let opts = ObjectOptions {
9878 no_lock: true,
9879 ..Default::default()
9880 };
9881
9882 store.pools[0]
9883 .make_bucket(bucket, &MakeBucketOptions::default())
9884 .await
9885 .expect("bucket should be created");
9886 let mut put_reader = PutObjReader::from_vec(payload.clone());
9887 store.pools[0]
9888 .put_object(bucket, object, &mut put_reader, &opts)
9889 .await
9890 .expect("object should be written");
9891
9892 clear_get_object_body_cache_hook();
9893 let hook = Arc::new(CountingMissHook {
9894 calls: AtomicUsize::new(0),
9895 });
9896 register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
9897 let _hook_guard = BodyCacheHookGuard;
9898
9899 let prepared = store
9900 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
9901 .await
9902 .expect("prepared reader metadata should resolve");
9903 assert!(matches!(
9904 lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
9905 GetObjectBodyCacheHookLookup::Miss
9906 ));
9907 assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");
9908
9909 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9910 assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
9911 let mut restored = Vec::new();
9912 reader
9913 .stream
9914 .read_to_end(&mut restored)
9915 .await
9916 .expect("prepared body should stream");
9917
9918 assert_eq!(restored, payload);
9919 assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
9920 }
9921
9922 #[tokio::test]
9923 #[serial_test::serial(body_cache_hook)]
9924 async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
9925 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
9926 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9927 let store = new_prepared_reader_test_store(&[set_disks]).await;
9928 let bucket = "prepared-reader-lock-lifetime";
9929 let object = "object.bin";
9930 let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
9931 let put_opts = ObjectOptions {
9932 no_lock: true,
9933 ..Default::default()
9934 };
9935
9936 store.pools[0]
9937 .make_bucket(bucket, &MakeBucketOptions::default())
9938 .await
9939 .expect("bucket should be created");
9940 let mut put_reader = PutObjReader::from_vec(payload.clone());
9941 store.pools[0]
9942 .put_object(bucket, object, &mut put_reader, &put_opts)
9943 .await
9944 .expect("object should be written");
9945
9946 let prepared = store
9947 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9948 .await
9949 .expect("prepared reader metadata should resolve");
9950 assert!(prepared.read_lock_guard.is_some());
9951 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9952
9953 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9954 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9955 let mut restored = Vec::new();
9956 reader
9957 .stream
9958 .read_to_end(&mut restored)
9959 .await
9960 .expect("prepared body should stream");
9961 assert_eq!(restored, payload);
9962 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9963
9964 let prepared = store
9965 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9966 .await
9967 .expect("second prepared reader metadata should resolve");
9968 let reader = prepared.into_reader().await.expect("second prepared body reader should open");
9969 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9970 drop(reader);
9971 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9972 })
9973 .await;
9974 }
9975
9976 #[tokio::test]
9977 #[serial_test::serial(body_cache_hook)]
9978 async fn select_snapshot_holds_namespace_lock_independent_of_get_optimization() {
9979 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
9980 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9981 let store = new_prepared_reader_test_store(&[set_disks]).await;
9982 let bucket = "select-snapshot-lock-lifetime";
9983 let object = "object.bin";
9984 let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
9985 let put_opts = ObjectOptions {
9986 no_lock: true,
9987 ..Default::default()
9988 };
9989
9990 store.pools[0]
9991 .make_bucket(bucket, &MakeBucketOptions::default())
9992 .await
9993 .expect("bucket should be created");
9994 let mut put_reader = PutObjReader::from_vec(payload.clone());
9995 store.pools[0]
9996 .put_object(bucket, object, &mut put_reader, &put_opts)
9997 .await
9998 .expect("object should be written");
9999
10000 use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
10001 let mut request_headers = HeaderMap::new();
10002 request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
10003 request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
10004 request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
10005 request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
10006 let snapshot = store
10007 .prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
10008 .await
10009 .expect("SelectObjectContent snapshot should be prepared");
10010 assert_eq!(snapshot.headers.len(), 3);
10011 assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
10012 assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
10013 assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
10014 assert!(snapshot.headers.get("authorization").is_none());
10015 assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
10016 assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
10017 assert_eq!(
10018 snapshot.logical_size(),
10019 u64::try_from(payload.len()).expect("test payload length should fit in u64")
10020 );
10021 assert_eq!(
10022 snapshot.object_info().size,
10023 i64::try_from(payload.len()).expect("test payload length should fit in i64")
10024 );
10025 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10026
10027 let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10028 let mut restored = Vec::new();
10029 reader
10030 .stream
10031 .read_to_end(&mut restored)
10032 .await
10033 .expect("snapshot body should stream");
10034 assert_eq!(restored, payload);
10035 drop(reader);
10036 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10037
10038 drop(snapshot);
10039 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10040 })
10041 .await;
10042 }
10043
10044 #[tokio::test]
10045 async fn select_snapshot_identity_compares_encoded_directory_object_key() {
10046 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10047 let store = new_prepared_reader_test_store(&[set_disks]).await;
10048 let bucket = "select-snapshot-directory-identity";
10049 let source_object = "source.bin";
10050
10051 store.pools[0]
10052 .make_bucket(bucket, &MakeBucketOptions::default())
10053 .await
10054 .expect("bucket should be created");
10055 let mut put_reader = PutObjReader::from_vec(b"source".to_vec());
10056 store.pools[0]
10057 .put_object(
10058 bucket,
10059 source_object,
10060 &mut put_reader,
10061 &ObjectOptions {
10062 no_lock: true,
10063 ..Default::default()
10064 },
10065 )
10066 .await
10067 .expect("source object should be written");
10068
10069 let mut snapshot = store
10070 .prepare_select_object_snapshot(bucket, source_object, &HeaderMap::new(), &ObjectOptions::default())
10071 .await
10072 .expect("source object snapshot should be prepared");
10073 snapshot.object = encode_dir_object("directory/");
10074
10075 assert!(snapshot.is_for(bucket, "directory/"));
10076 assert!(!snapshot.is_for(bucket, "different/"));
10077 }
10078
10079 #[tokio::test]
10080 #[serial_test::serial(body_cache_hook)]
10081 async fn select_snapshot_reuses_initial_metadata_fanout_for_first_reader() {
10082 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10083 let store = new_prepared_reader_test_store(&[set_disks]).await;
10084 let bucket = "select-snapshot-initial-metadata";
10085 let object = "initial-metadata.bin";
10086 let payload = b"select snapshot initial metadata".repeat(4_000);
10087 let no_lock_opts = ObjectOptions {
10088 no_lock: true,
10089 ..Default::default()
10090 };
10091
10092 store.pools[0]
10093 .make_bucket(bucket, &MakeBucketOptions::default())
10094 .await
10095 .expect("bucket should be created");
10096 let mut put_reader = PutObjReader::from_vec(payload.clone());
10097 store.pools[0]
10098 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10099 .await
10100 .expect("object should be written");
10101
10102 let calls = disk_call_counters::observe(object);
10103 let snapshot = store
10104 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10105 .await
10106 .expect("snapshot should prepare metadata");
10107 assert_eq!(
10108 calls.total(disk_call_counters::KIND_READ_VERSION),
10109 4,
10110 "snapshot preparation should fan out metadata once"
10111 );
10112
10113 let mut reader = snapshot.open_reader(None).await.expect("first snapshot reader should open");
10114 assert_eq!(
10115 calls.total(disk_call_counters::KIND_READ_VERSION),
10116 4,
10117 "first reader should consume the metadata captured during preparation"
10118 );
10119 let mut restored = Vec::new();
10120 reader
10121 .stream
10122 .read_to_end(&mut restored)
10123 .await
10124 .expect("snapshot body should stream");
10125 assert_eq!(restored, payload);
10126 }
10127
10128 #[tokio::test]
10129 #[serial_test::serial(body_cache_hook)]
10130 async fn select_snapshot_rejects_identity_change_before_second_reader() {
10131 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10132 let store = new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await;
10133 let bucket = "select-snapshot-identity-change";
10134 let object = "identity-change.bin";
10135 let no_lock_opts = ObjectOptions {
10136 no_lock: true,
10137 ..Default::default()
10138 };
10139
10140 set_disks
10141 .make_bucket(bucket, &MakeBucketOptions::default())
10142 .await
10143 .expect("bucket should be created");
10144 let mut initial_reader = PutObjReader::from_vec(b"first generation".to_vec());
10145 set_disks
10146 .put_object(bucket, object, &mut initial_reader, &no_lock_opts)
10147 .await
10148 .expect("initial generation should be written");
10149 let mut snapshot = store
10150 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10151 .await
10152 .expect("snapshot should be prepared");
10153 drop(
10154 snapshot
10155 .open_reader(None)
10156 .await
10157 .expect("first snapshot reader should consume captured metadata"),
10158 );
10159 snapshot.opts.metadata_cache_safe = false;
10160
10161 let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10162 set_disks
10163 .put_object(bucket, object, &mut replacement_reader, &no_lock_opts)
10164 .await
10165 .expect("test-only no-lock write should replace the object");
10166 let error = match snapshot.open_reader(None).await {
10167 Ok(_) => panic!("a later reader must reject the replacement generation"),
10168 Err(error) => error,
10169 };
10170 assert!(matches!(
10171 error,
10172 SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged)
10173 ));
10174 }
10175
10176 #[tokio::test]
10177 #[serial_test::serial(body_cache_hook)]
10178 async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() {
10179 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10180 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10181 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10182 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10183 let bucket = "select-snapshot-latest-delete-marker";
10184 let object = "versioned-object.bin";
10185 let versioned_opts = ObjectOptions {
10186 no_lock: true,
10187 versioned: true,
10188 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10189 ..Default::default()
10190 };
10191
10192 for set_disks in [&first_set, &second_set] {
10193 set_disks
10194 .make_bucket(bucket, &MakeBucketOptions::default())
10195 .await
10196 .expect("bucket should be created");
10197 }
10198 let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10199 first_set
10200 .put_object(bucket, object, &mut older_reader, &versioned_opts)
10201 .await
10202 .expect("older versioned object should be written");
10203 let mut put_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10204 second_set
10205 .put_object(bucket, object, &mut put_reader, &versioned_opts)
10206 .await
10207 .expect("versioned object should be written");
10208 let marker = second_set
10209 .delete_object(bucket, object, versioned_opts.clone())
10210 .await
10211 .expect("versioned delete should create a marker");
10212 assert!(marker.delete_marker);
10213 assert!(marker.version_id.is_some_and(|version_id| !version_id.is_nil()));
10214
10215 let error = store
10216 .prepare_select_object_snapshot(
10217 bucket,
10218 object,
10219 &HeaderMap::new(),
10220 &ObjectOptions {
10221 versioned: true,
10222 ..Default::default()
10223 },
10224 )
10225 .await
10226 .expect_err("latest delete marker should hide the prior object generation");
10227 assert!(matches!(
10228 error,
10229 PrepareSelectObjectSnapshotError::Storage(ref error) if is_err_object_not_found(error)
10230 ));
10231 }
10232
10233 #[tokio::test]
10234 #[serial_test::serial(storage_class_env)]
10235 async fn select_snapshot_blocks_store_delete_for_object_in_nonzero_set() {
10236 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10237 let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
10238 ctx.update_erasure_type(SetupType::DistErasure).await;
10239 assert!(
10240 sets.disk_set[0]
10241 .lockers
10242 .iter()
10243 .all(|first| sets.disk_set[1].lockers.iter().all(|second| !Arc::ptr_eq(first, second)))
10244 );
10245 let pool_config = sets.endpoints.clone();
10246 let store = Arc::new(new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx));
10247 let bucket = RUSTFS_META_BUCKET;
10248 let object = (0..1_000)
10249 .map(|index| format!("nonzero-set-{index}.bin"))
10250 .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
10251 .expect("a key should hash to the second set");
10252 let mut put_reader = PutObjReader::from_vec(b"stable snapshot body".to_vec());
10253 sets.put_object(
10254 bucket,
10255 &object,
10256 &mut put_reader,
10257 &ObjectOptions {
10258 no_lock: true,
10259 ..Default::default()
10260 },
10261 )
10262 .await
10263 .expect("object should be written to the second set");
10264 assert!(Arc::ptr_eq(&sets.get_disks_by_key(&object), &sets.disk_set[1]));
10265
10266 let snapshot = store
10267 .prepare_select_object_snapshot(bucket, &object, &HeaderMap::new(), &ObjectOptions::default())
10268 .await
10269 .expect("snapshot should acquire both lock domains");
10270 let hashed_set_writer = rustfs_lock::NamespaceLock::with_clients_and_quorum(
10271 "select-nonzero-set-writer".to_string(),
10272 sets.disk_set[1].lockers.clone(),
10273 2,
10274 );
10275 let writer_error = hashed_set_writer
10276 .get_write_lock(
10277 rustfs_lock::ObjectKey::new(bucket, object.as_str()),
10278 "competing-hashed-set-writer",
10279 Duration::from_millis(50),
10280 )
10281 .await
10282 .expect_err("Select must hold the nonzero hashed-set read lock");
10283 assert!(matches!(writer_error, rustfs_lock::LockError::Timeout { .. }));
10284 let barrier = DeleteAfterObjectLockSnapshotBarrier::install(bucket);
10285 let delete_store = Arc::clone(&store);
10286 let delete_object = object.clone();
10287 let mut delete = tokio::spawn(async move {
10288 delete_store
10289 .delete_object(bucket, &delete_object, ObjectOptions::default())
10290 .await
10291 });
10292 barrier.wait_until_paused().await;
10293 barrier.release();
10294 assert!(
10295 tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(),
10296 "store DELETE must wait for the fixed-domain Select read lock"
10297 );
10298
10299 drop(snapshot);
10300 let deleted = tokio::time::timeout(Duration::from_secs(10), delete)
10301 .await
10302 .expect("DELETE should resume after the snapshot is dropped")
10303 .expect("DELETE task should not panic")
10304 .expect("DELETE should complete");
10305 assert_eq!(deleted.name, object);
10306 }
10307
10308 #[tokio::test]
10309 #[serial_test::serial]
10310 async fn select_snapshot_fails_closed_when_local_locking_is_disabled() {
10311 let ctx = Arc::new(crate::runtime::instance::InstanceContext::with_lock_manager_for_test(Arc::new(
10312 rustfs_lock::GlobalLockManager::Disabled(rustfs_lock::DisabledLockManager::new()),
10313 )));
10314 let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10315 let store = new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await;
10316 let bucket = "select-snapshot-lock-disabled";
10317 let object = "object.bin";
10318 let mut put_reader = PutObjReader::from_vec(b"payload".to_vec());
10319 let no_lock_opts = ObjectOptions {
10320 no_lock: true,
10321 ..Default::default()
10322 };
10323
10324 store.pools[0]
10325 .make_bucket(bucket, &MakeBucketOptions::default())
10326 .await
10327 .expect("bucket should be created");
10328 store.pools[0]
10329 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10330 .await
10331 .expect("object should be written without namespace locking");
10332
10333 let error = store
10334 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10335 .await
10336 .expect_err("SelectObjectContent must reject a disabled lock manager");
10337 assert!(matches!(
10338 error,
10339 PrepareSelectObjectSnapshotError::Consistency(SnapshotConsistencyError::LockingDisabled)
10340 ));
10341 }
10342
10343 #[tokio::test]
10344 #[serial_test::serial(body_cache_hook)]
10345 async fn prepared_object_info_releases_namespace_lock_immediately() {
10346 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10347 let store = new_prepared_reader_test_store(&[set_disks]).await;
10348 let bucket = "prepared-object-info-lock-release";
10349 let object = "object.bin";
10350 let payload = b"prepared-object-info-lock-release".to_vec();
10351 let put_opts = ObjectOptions {
10352 no_lock: true,
10353 ..Default::default()
10354 };
10355
10356 store.pools[0]
10357 .make_bucket(bucket, &MakeBucketOptions::default())
10358 .await
10359 .expect("bucket should be created");
10360 let mut put_reader = PutObjReader::from_vec(payload.clone());
10361 store.pools[0]
10362 .put_object(bucket, object, &mut put_reader, &put_opts)
10363 .await
10364 .expect("object should be written");
10365
10366 let prepared = store
10367 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
10368 .await
10369 .expect("prepared reader metadata should resolve");
10370 assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10371 assert_eq!(prepared.into_object_info().size, payload.len() as i64);
10372
10373 drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10374 }
10375
10376 #[tokio::test]
10377 #[serial_test::serial(body_cache_hook)]
10378 async fn prepared_reader_reuses_metadata_across_three_pools() {
10379 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10380 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10381 let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10382 let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10383 let bucket = "prepared-reader-three-pools";
10384 let object = "object.bin";
10385 let payload = b"prepared-reader-three-pool-payload-".repeat(40_000);
10386 let opts = ObjectOptions {
10387 no_lock: true,
10388 ..Default::default()
10389 };
10390
10391 for pool in &store.pools {
10392 pool.make_bucket(bucket, &MakeBucketOptions::default())
10393 .await
10394 .expect("bucket should be created in each pool");
10395 }
10396 let mut put_reader = PutObjReader::from_vec(payload.clone());
10397 store.pools[2]
10398 .put_object(bucket, object, &mut put_reader, &opts)
10399 .await
10400 .expect("object should be written only to the third pool");
10401
10402 let calls = disk_call_counters::observe(object);
10403 let prepared = store
10404 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10405 .await
10406 .expect("prepared reader should resolve the third-pool object");
10407 assert_eq!(prepared.object_info().size, payload.len() as i64);
10408 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10409 assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10410 let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
10411 assert_eq!(
10412 calls.total(disk_call_counters::KIND_READ_VERSION),
10413 metadata_calls,
10414 "the selected pool must reuse its prepared metadata"
10415 );
10416 let mut restored = Vec::new();
10417 reader
10418 .stream
10419 .read_to_end(&mut restored)
10420 .await
10421 .expect("prepared body should stream");
10422 assert_eq!(restored, payload);
10423 }
10424
10425 #[tokio::test]
10426 #[serial_test::serial(body_cache_hook)]
10427 async fn select_snapshot_reuses_metadata_across_three_pools() {
10428 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10429 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10430 let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10431 let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10432 let bucket = "select-snapshot-three-pools";
10433 let object = "object.bin";
10434 let payload = b"select-snapshot-three-pool-payload-".repeat(40_000);
10435 let write_opts = ObjectOptions {
10436 no_lock: true,
10437 ..Default::default()
10438 };
10439
10440 for pool in &store.pools {
10441 pool.make_bucket(bucket, &MakeBucketOptions::default())
10442 .await
10443 .expect("bucket should be created in each pool");
10444 }
10445 let mut put_reader = PutObjReader::from_vec(payload.clone());
10446 store.pools[2]
10447 .put_object(bucket, object, &mut put_reader, &write_opts)
10448 .await
10449 .expect("object should be written only to the third pool");
10450
10451 let calls = disk_call_counters::observe(object);
10452 let snapshot = store
10453 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10454 .await
10455 .expect("SelectObjectContent snapshot should resolve the third-pool object");
10456 assert_eq!(snapshot.object_info().size, payload.len() as i64);
10457 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10458 assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10459
10460 let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open");
10461 assert_eq!(
10462 calls.total(disk_call_counters::KIND_READ_VERSION),
10463 metadata_calls,
10464 "SelectObjectContent must consume the prepared winner without a second fanout"
10465 );
10466 let mut restored = Vec::new();
10467 reader
10468 .stream
10469 .read_to_end(&mut restored)
10470 .await
10471 .expect("snapshot body should stream");
10472 assert_eq!(restored, payload);
10473 }
10474
10475 #[tokio::test]
10476 #[serial_test::serial(body_cache_hook)]
10477 async fn legacy_reader_reuses_selected_pool_metadata() {
10478 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10479 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10480 let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10481 let bucket = "legacy-reader-second-pool";
10482 let object = "object.bin";
10483 let payload = b"legacy-reader-second-pool-payload-".repeat(40_000);
10484 let opts = ObjectOptions {
10485 no_lock: true,
10486 ..Default::default()
10487 };
10488
10489 for pool in &store.pools {
10490 pool.make_bucket(bucket, &MakeBucketOptions::default())
10491 .await
10492 .expect("bucket should be created in each pool");
10493 }
10494 let mut put_reader = PutObjReader::from_vec(payload.clone());
10495 store.pools[1]
10496 .put_object(bucket, object, &mut put_reader, &opts)
10497 .await
10498 .expect("object should be written only to the second pool");
10499
10500 clear_get_object_body_cache_hook();
10501 let hook = Arc::new(CountingMissHook {
10502 calls: AtomicUsize::new(0),
10503 });
10504 register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
10505 let _hook_guard = BodyCacheHookGuard;
10506
10507 let calls = disk_call_counters::observe(object);
10508 let mut reader = store
10509 .handle_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10510 .await
10511 .expect("legacy reader should resolve the second-pool object");
10512 assert_eq!(
10513 hook.calls.load(Ordering::Relaxed),
10514 1,
10515 "legacy reader must probe the body cache exactly once"
10516 );
10517 assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
10518 assert!(
10519 calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10520 "legacy reader must fan out each 4-disk pool at most once"
10521 );
10522 let mut restored = Vec::new();
10523 reader
10524 .stream
10525 .read_to_end(&mut restored)
10526 .await
10527 .expect("legacy reader body should stream");
10528 assert_eq!(restored, payload);
10529 }
10530
10531 fn prepared_pool_test_status(id: usize, suspended: bool) -> PoolStatus {
10532 PoolStatus {
10533 id,
10534 cmd_line: format!("prepared-pool-{id}"),
10535 last_update: OffsetDateTime::now_utc(),
10536 decommission: suspended.then(|| PoolDecommissionInfo {
10537 start_time: Some(OffsetDateTime::now_utc()),
10538 ..Default::default()
10539 }),
10540 }
10541 }
10542
10543 #[tokio::test]
10544 #[serial_test::serial(body_cache_hook)]
10545 async fn prepared_reader_refetches_when_final_pool_state_changes_winner() {
10546 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10547 let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10548 let bucket = "prepared-reader-pool-state-fallback";
10549 let object = "object.bin";
10550 let payload = b"pool-state fallback payload".repeat(8_000);
10551 let opts = ObjectOptions {
10552 no_lock: true,
10553 ..Default::default()
10554 };
10555
10556 set_disks
10557 .make_bucket(bucket, &MakeBucketOptions::default())
10558 .await
10559 .expect("bucket should be created");
10560 let mut put_reader = PutObjReader::from_vec(payload.clone());
10561 set_disks
10562 .put_object(bucket, object, &mut put_reader, &opts)
10563 .await
10564 .expect("shared object should be written");
10565
10566 let calls = disk_call_counters::observe(object);
10567 let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, false);
10568 let read_store = Arc::clone(&store);
10569 let read_opts = opts.clone();
10570 let read = tokio::spawn(async move {
10571 read_store
10572 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10573 .await
10574 });
10575 barrier.wait_after_fanout().await;
10576 *store.pool_meta.write().await = PoolMeta {
10577 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10578 ..Default::default()
10579 };
10580 barrier.release_after_fanout();
10581
10582 let prepared = read
10583 .await
10584 .expect("prepared read task should not panic")
10585 .expect("final active pool should be refetched");
10586 assert!(Arc::ptr_eq(&prepared.pool, &store.pools[0]));
10587 assert_eq!(
10588 calls.total(disk_call_counters::KIND_READ_VERSION),
10589 12,
10590 "two initial 4-disk fanouts plus one fallback refetch are required"
10591 );
10592 let mut reader = prepared.into_reader().await.expect("fallback body reader should open");
10593 let mut restored = Vec::new();
10594 reader
10595 .stream
10596 .read_to_end(&mut restored)
10597 .await
10598 .expect("fallback body should stream");
10599 assert_eq!(restored, payload);
10600 }
10601
10602 #[tokio::test]
10603 #[serial_test::serial(body_cache_hook)]
10604 async fn prepared_reader_fallback_rejects_generation_change_before_refetch() {
10605 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10606 let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10607 let bucket = "prepared-reader-pool-state-generation-change";
10608 let object = "object.bin";
10609 let opts = ObjectOptions {
10610 no_lock: true,
10611 ..Default::default()
10612 };
10613
10614 set_disks
10615 .make_bucket(bucket, &MakeBucketOptions::default())
10616 .await
10617 .expect("bucket should be created");
10618 let mut initial_reader = PutObjReader::from_vec(b"initial generation".to_vec());
10619 set_disks
10620 .put_object(bucket, object, &mut initial_reader, &opts)
10621 .await
10622 .expect("initial object should be written");
10623
10624 let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, true);
10625 let read_store = Arc::clone(&store);
10626 let read_opts = opts.clone();
10627 let read = tokio::spawn(async move {
10628 read_store
10629 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10630 .await
10631 });
10632 barrier.wait_after_fanout().await;
10633 *store.pool_meta.write().await = PoolMeta {
10634 pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10635 ..Default::default()
10636 };
10637 barrier.release_after_fanout();
10638 barrier.wait_before_refetch().await;
10639
10640 let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10641 set_disks
10642 .put_object(bucket, object, &mut replacement_reader, &opts)
10643 .await
10644 .expect("replacement generation should be written before fallback refetch");
10645 barrier.release_before_refetch();
10646
10647 let error = match read.await.expect("prepared read task should not panic") {
10648 Ok(_) => panic!("changed fallback generation must not be accepted"),
10649 Err(error) => error,
10650 };
10651 assert_eq!(error, Error::ErasureReadQuorum);
10652 }
10653
10654 #[tokio::test]
10655 #[serial_test::serial(body_cache_hook)]
10656 async fn prepared_reader_rejects_latest_delete_marker_without_refetching_metadata() {
10657 let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10658 let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10659 let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10660 let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10661 let bucket = "prepared-reader-latest-delete-marker";
10662 let object = "versioned-object.bin";
10663 let versioned_opts = ObjectOptions {
10664 no_lock: true,
10665 versioned: true,
10666 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10667 ..Default::default()
10668 };
10669
10670 for set_disks in [&first_set, &second_set] {
10671 set_disks
10672 .make_bucket(bucket, &MakeBucketOptions::default())
10673 .await
10674 .expect("bucket should be created");
10675 }
10676 let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10677 first_set
10678 .put_object(bucket, object, &mut older_reader, &versioned_opts)
10679 .await
10680 .expect("older object should be written");
10681 let mut hidden_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10682 second_set
10683 .put_object(bucket, object, &mut hidden_reader, &versioned_opts)
10684 .await
10685 .expect("newer object should be written");
10686 let marker = second_set
10687 .delete_object(bucket, object, versioned_opts.clone())
10688 .await
10689 .expect("delete marker should be committed");
10690 assert!(marker.delete_marker);
10691
10692 let calls = disk_call_counters::observe(object);
10693 let error = match store
10694 .prepare_get_object_reader(
10695 bucket,
10696 object,
10697 None,
10698 HeaderMap::new(),
10699 &ObjectOptions {
10700 no_lock: true,
10701 versioned: true,
10702 ..Default::default()
10703 },
10704 )
10705 .await
10706 {
10707 Ok(_) => panic!("latest delete marker should hide the older live object"),
10708 Err(error) => error,
10709 };
10710
10711 assert!(is_err_object_not_found(&error));
10712 assert!(
10713 calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10714 "delete-marker resolution must fan out each pool at most once"
10715 );
10716 }
10717
10718 #[tokio::test]
10719 #[serial_test::serial(body_cache_hook)]
10720 async fn prepared_reader_explicit_version_reuses_the_matching_pool_metadata() {
10721 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10722 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10723 let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await;
10724 let bucket = "prepared-reader-explicit-version";
10725 let object = "versioned-object.bin";
10726 let payload = b"explicit version from first pool".repeat(8_000);
10727 let versioned_opts = ObjectOptions {
10728 no_lock: true,
10729 versioned: true,
10730 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10731 ..Default::default()
10732 };
10733
10734 for set_disks in [&first_set, &second_set] {
10735 set_disks
10736 .make_bucket(bucket, &MakeBucketOptions::default())
10737 .await
10738 .expect("bucket should be created");
10739 }
10740 let mut first_reader = PutObjReader::from_vec(payload.clone());
10741 let first = first_set
10742 .put_object(bucket, object, &mut first_reader, &versioned_opts)
10743 .await
10744 .expect("requested version should be written to the first pool");
10745 let mut second_reader = PutObjReader::from_vec(b"different pool version".to_vec());
10746 second_set
10747 .put_object(bucket, object, &mut second_reader, &versioned_opts)
10748 .await
10749 .expect("a different version should be written to the second pool");
10750
10751 let requested_version = first
10752 .version_id
10753 .expect("versioned PUT should return a version id")
10754 .to_string();
10755 let read_opts = ObjectOptions {
10756 no_lock: true,
10757 versioned: true,
10758 version_id: Some(requested_version),
10759 ..Default::default()
10760 };
10761 let calls = disk_call_counters::observe(object);
10762 let prepared = store
10763 .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10764 .await
10765 .expect("explicit version should resolve from the matching pool");
10766 assert_eq!(prepared.object_info().version_id, first.version_id);
10767 let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10768 assert!(metadata_calls <= 8, "explicit-version lookup must fan out each pool at most once");
10769
10770 let mut reader = prepared
10771 .into_reader()
10772 .await
10773 .expect("prepared explicit-version body should open");
10774 assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), metadata_calls);
10775 let mut restored = Vec::new();
10776 reader
10777 .stream
10778 .read_to_end(&mut restored)
10779 .await
10780 .expect("explicit-version body should stream");
10781 assert_eq!(restored, payload);
10782 }
10783
10784 #[tokio::test]
10785 #[serial_test::serial]
10786 async fn select_snapshot_locks_the_hashed_set_in_every_pool() {
10787 let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10788 let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10789 let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10790 let bucket = "select-snapshot-all-pools";
10791 let object = "object.bin";
10792 let payload = b"second-pool-snapshot".to_vec();
10793 let no_lock_opts = ObjectOptions {
10794 no_lock: true,
10795 ..Default::default()
10796 };
10797
10798 for pool in &store.pools {
10799 pool.make_bucket(bucket, &MakeBucketOptions::default())
10800 .await
10801 .expect("bucket should be created in each pool");
10802 }
10803 let mut put_reader = PutObjReader::from_vec(payload.clone());
10804 store.pools[1]
10805 .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10806 .await
10807 .expect("object should be written only to the second pool");
10808
10809 let snapshot = store
10810 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10811 .await
10812 .expect("snapshot should resolve the second-pool object");
10813 assert_pool_writer_is_blocked(&store, 0, bucket, object).await;
10814 assert_pool_writer_is_blocked(&store, 1, bucket, object).await;
10815
10816 let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10817 let mut restored = Vec::new();
10818 reader
10819 .stream
10820 .read_to_end(&mut restored)
10821 .await
10822 .expect("snapshot body should stream");
10823 assert_eq!(restored, payload);
10824 drop(reader);
10825 drop(snapshot);
10826
10827 drop(acquire_pool_writer(&store, 0, bucket, object).await);
10828 drop(acquire_pool_writer(&store, 1, bucket, object).await);
10829 }
10830
10831 #[tokio::test]
10835 async fn instance_context_flows_through_object_graph() {
10836 let store = new_read_lock_test_store().await;
10837
10838 let sets = store.pools.first().expect("test store has one pool");
10839 assert!(
10840 std::sync::Arc::ptr_eq(&store.ctx, sets.instance_ctx()),
10841 "Sets must carry the store's instance context"
10842 );
10843
10844 let set_disks = sets.disk_set.first().expect("pool has one set");
10845 assert!(
10846 std::sync::Arc::ptr_eq(sets.instance_ctx(), set_disks.instance_ctx()),
10847 "SetDisks must carry the Sets' instance context"
10848 );
10849 }
10850
10851 #[tokio::test]
10856 async fn set_disks_lock_manager_comes_from_instance_context() {
10857 let store = new_read_lock_test_store().await;
10858 let set_disks = store.pools[0].disk_set.first().expect("pool has one set");
10859
10860 assert!(
10861 std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &set_disks.instance_ctx().lock_manager()),
10862 "SetDisks lock manager must be sourced from its instance context"
10863 );
10864 assert!(
10865 std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &rustfs_lock::get_global_lock_manager()),
10866 "single-instance lock manager must alias the process singleton"
10867 );
10868 }
10869
10870 #[tokio::test]
10871 #[serial_test::serial]
10872 async fn delete_objects_write_locks_cover_each_unique_object() {
10873 let store = new_read_lock_test_store().await;
10874 let objects = vec![
10875 ObjectToDelete {
10876 object_name: "beta".to_string(),
10877 ..Default::default()
10878 },
10879 ObjectToDelete {
10880 object_name: "alpha".to_string(),
10881 ..Default::default()
10882 },
10883 ObjectToDelete {
10884 object_name: "beta".to_string(),
10885 ..Default::default()
10886 },
10887 ];
10888 let mut opts = ObjectOptions::default();
10889
10890 let guards = store
10891 .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
10892 .await
10893 .expect("delete object locks should be acquired");
10894
10895 assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
10896 assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
10897 assert!(
10898 opts.namespace_lock_fence.is_some(),
10899 "set layer must receive the outer write-lock loss fence"
10900 );
10901
10902 let alpha_lock = store
10903 .handle_new_ns_lock("bucket", "alpha")
10904 .await
10905 .expect("alpha namespace lock should be created");
10906 let err = alpha_lock
10907 .get_read_lock(Duration::from_millis(20))
10908 .await
10909 .expect_err("batch delete write guard should block alpha readers");
10910 assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
10911
10912 drop(guards);
10913 alpha_lock
10914 .get_read_lock(Duration::from_secs(1))
10915 .await
10916 .expect("alpha read lock should be available after dropping batch guards");
10917 }
10918
10919 #[tokio::test]
10920 async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
10921 let store = new_read_lock_test_store().await;
10922 let mut opts = ObjectOptions::default();
10923
10924 let guard = store
10925 .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10926 .await
10927 .expect("read lock should be acquired");
10928
10929 assert!(guard.is_some(), "read lock should be held by the outer store layer");
10930 assert!(opts.no_lock, "set layer should not reacquire the object lock");
10931 assert!(
10932 opts.metadata_cache_safe,
10933 "metadata cache is safe only because the outer store layer acquired the read lock"
10934 );
10935 }
10936
10937 #[tokio::test]
10938 async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
10939 let store = new_read_lock_test_store().await;
10940 let mut opts = ObjectOptions {
10941 no_lock: true,
10942 ..Default::default()
10943 };
10944
10945 let guard = store
10946 .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10947 .await
10948 .expect("prelocked read should not acquire another lock");
10949
10950 assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
10951 assert!(
10952 !opts.metadata_cache_safe,
10953 "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
10954 );
10955 }
10956
10957 #[tokio::test]
10967 #[serial_test::serial]
10968 async fn restore_accept_guard_serializes_concurrent_accepts() {
10969 let store = Arc::new(new_read_lock_test_store().await);
10973 let _first = store
10974 .acquire_restore_accept_guard("bucket", "object")
10975 .await
10976 .expect("first accept guard should be acquired");
10977
10978 let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
10979 match store.acquire_restore_accept_guard("bucket", "object").await {
10980 Ok(_) => panic!("second accept guard must wait behind the first"),
10981 Err(err) => err,
10982 }
10983 })
10984 .await;
10985
10986 assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
10987 }
10988
10989 #[tokio::test]
10990 #[serial_test::serial]
10991 async fn restore_worker_guard_proves_generation_liveness_until_drop() {
10992 let store = Arc::new(new_read_lock_test_store().await);
10993 let operation_id = Uuid::from_u128(1);
10994
10995 let first = store
10996 .acquire_restore_worker_guard(operation_id)
10997 .await
10998 .expect("first worker must own its newly generated lock");
10999 assert!(
11000 store
11001 .try_acquire_restore_worker_guard(operation_id)
11002 .await
11003 .expect("a contended liveness probe should remain typed")
11004 .is_none(),
11005 "the same generation must remain live while its worker guard is held"
11006 );
11007
11008 drop(first);
11009 assert!(
11010 store
11011 .try_acquire_restore_worker_guard(operation_id)
11012 .await
11013 .expect("post-drop liveness probe should not fail")
11014 .is_some(),
11015 "dropping the worker guard must make the orphan generation recoverable"
11016 );
11017 }
11018
11019 #[tokio::test]
11020 #[serial_test::serial]
11021 async fn reader_lock_is_held_when_optimization_is_disabled() {
11022 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
11023 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11024 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11025 let key = rustfs_lock::ObjectKey::new("bucket", "object");
11026 let read_guard = lock
11027 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11028 .await
11029 .expect("read lock should be acquired");
11030 let read_guard = ObjectLockDiagGuard::new(
11031 read_guard,
11032 true,
11033 "test_get_object",
11034 Some("bucket".to_string()),
11035 Some("object".to_string()),
11036 Some("reader".to_string()),
11037 ObjectLockDiagMode::Read,
11038 );
11039 let reader = GetObjectReader {
11040 stream: Box::new(Cursor::new(Vec::<u8>::new())),
11041 object_info: ObjectInfo::default(),
11042 buffered_body: None,
11043 body_source: Default::default(),
11044 };
11045
11046 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11047
11048 lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
11049 .await
11050 .expect_err("reader should hold the read lock");
11051 drop(reader);
11052 lock.get_write_lock(key, "writer", Duration::from_secs(1))
11053 .await
11054 .expect("dropping the reader should release the read lock");
11055 })
11056 .await;
11057 }
11058
11059 #[tokio::test]
11060 #[serial_test::serial]
11061 async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
11062 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
11063 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11064 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11065 let key = rustfs_lock::ObjectKey::new("bucket", "object");
11066 let read_guard = lock
11067 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11068 .await
11069 .expect("read lock should be acquired");
11070 let read_guard = ObjectLockDiagGuard::new(
11071 read_guard,
11072 true,
11073 "test_get_object",
11074 Some("bucket".to_string()),
11075 Some("object".to_string()),
11076 Some("reader".to_string()),
11077 ObjectLockDiagMode::Read,
11078 );
11079 let reader = GetObjectReader {
11080 stream: Box::new(Cursor::new(vec![1, 2, 3])),
11081 object_info: ObjectInfo::default(),
11082 buffered_body: None,
11083 body_source: Default::default(),
11084 };
11085
11086 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11087
11088 lock.get_write_lock(key, "writer", Duration::from_secs(1))
11089 .await
11090 .expect("lock optimization should release the read lock before returning the stream");
11091 drop(reader);
11092 })
11093 .await;
11094 }
11095
11096 #[tokio::test]
11097 #[serial_test::serial]
11098 async fn reader_lock_is_not_held_for_buffered_body_when_optimization_is_enabled() {
11099 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
11100 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11101 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11102 let key = rustfs_lock::ObjectKey::new("bucket", "object");
11103 let read_guard = lock
11104 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11105 .await
11106 .expect("read lock should be acquired");
11107 let read_guard = ObjectLockDiagGuard::new(
11108 read_guard,
11109 true,
11110 "test_get_object",
11111 Some("bucket".to_string()),
11112 Some("object".to_string()),
11113 Some("reader".to_string()),
11114 ObjectLockDiagMode::Read,
11115 );
11116 let reader = GetObjectReader {
11117 stream: Box::new(Cursor::new(vec![1, 2, 3])),
11118 object_info: ObjectInfo::default(),
11119 buffered_body: Some(Bytes::from_static(b"123")),
11120 body_source: Default::default(),
11121 };
11122
11123 let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11124
11125 lock.get_write_lock(key, "writer", Duration::from_secs(1))
11126 .await
11127 .expect("buffered reader should release the read lock immediately");
11128 drop(reader);
11129 })
11130 .await;
11131 }
11132
11133 #[tokio::test]
11134 #[serial_test::serial]
11135 async fn reader_lock_is_released_after_stream_eof() {
11136 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
11137 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
11138 let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
11139 let key = rustfs_lock::ObjectKey::new("bucket", "object");
11140 let read_guard = lock
11141 .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
11142 .await
11143 .expect("read lock should be acquired");
11144 let read_guard = ObjectLockDiagGuard::new(
11145 read_guard,
11146 true,
11147 "test_get_object",
11148 Some("bucket".to_string()),
11149 Some("object".to_string()),
11150 Some("reader".to_string()),
11151 ObjectLockDiagMode::Read,
11152 );
11153 let reader = GetObjectReader {
11154 stream: Box::new(Cursor::new(vec![1, 2, 3])),
11155 object_info: ObjectInfo::default(),
11156 buffered_body: None,
11157 body_source: Default::default(),
11158 };
11159
11160 let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
11161 let mut output = Vec::new();
11162 reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
11163 assert_eq!(output, vec![1, 2, 3]);
11164
11165 lock.get_write_lock(key, "writer", Duration::from_secs(1))
11166 .await
11167 .expect("EOF should release the read lock before the reader is dropped");
11168 drop(reader);
11169 })
11170 .await;
11171 }
11172}