#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX;
#[cfg(test)]
use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
#[cfg(test)]
use super::super::get_metadata_slowtail_fault_delay;
use super::super::{
Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion,
GetCodecStreamingFallbackReason, GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult,
HealChannelPriority, HealRequestSource, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext,
OBJECT_OP_IGNORED_ERRS, ObjectOptions, ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET,
RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks,
SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs,
can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
try_read_inline_data_shards_direct, warn,
};
#[cfg(test)]
pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
pub(in crate::set_disk) use super::metadata_quorum::{
MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches,
};
#[cfg(test)]
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
#[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH;
#[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD;
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING,
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled,
record_get_stage_duration_if_enabled,
};
#[cfg(test)]
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
};
#[cfg(test)]
use crate::disk::CHECK_PART_FILE_NOT_FOUND;
use crate::disk::DiskAPI;
#[cfg(test)]
use crate::disk::DiskOption;
#[cfg(test)]
use crate::disk::RUSTFS_META_TMP_BUCKET;
use crate::disk::disk_store::get_drive_metadata_timeout;
#[cfg(test)]
use crate::disk::endpoint::Endpoint;
#[cfg(test)]
use crate::disk::format::FormatV3;
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
#[cfg(test)]
use crate::disk::new_disk;
use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
use crate::io_support::bitrot::{
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
create_deferred_bitrot_reader_with_stripe_handle,
};
#[cfg(unix)]
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
use crate::set_disk::runtime_sources;
use crate::set_disk::shard_source::ShardReadCost;
use crate::storage_api_contracts::object::ObjectOperations;
use futures::FutureExt as _;
use futures::stream::{FuturesUnordered, StreamExt};
use metrics::counter;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP,
};
use std::{
collections::{HashMap, HashSet, VecDeque},
future::Future,
pin::Pin,
sync::{
Arc, OnceLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
time::{Duration, Instant},
};
fn metadata_metrics_path(bucket: &str) -> &'static str {
if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
GET_OBJECT_PATH_LEGACY_DUPLEX
}
}
fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
fn read_version_coalescing_enabled() -> bool {
let enabled = || {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
};
#[cfg(test)]
{
enabled()
}
#[cfg(not(test))]
{
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(enabled)
}
}
fn read_version_coalescing_delay() -> Duration {
#[cfg(test)]
{
let micros = rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
);
Duration::from_micros(micros)
}
#[cfg(not(test))]
{
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| {
Duration::from_micros(rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
))
})
}
}
struct CoalescedReadVersionRequest {
item: BatchReadVersionItem,
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ExpectedBatchReadVersionItem {
path: String,
version_id: String,
}
impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem {
fn from(item: &BatchReadVersionItem) -> Self {
Self {
path: item.path.clone(),
version_id: item.version_id.clone(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
incl_free_versions: bool,
read_data: bool,
healing: bool,
}
impl ReadVersionCoalescerKey {
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
Self {
disk: Arc::as_ptr(disk) as usize,
incl_free_versions: opts.incl_free_versions,
read_data: opts.read_data,
healing: opts.healing,
}
}
}
#[derive(Default)]
struct ReadVersionCoalescer {
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
}
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
}
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
counter!(
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
"event" => event,
"item_count" => item_count.to_string()
)
.increment(1);
}
fn batch_read_version_stage_timer() -> Option<Instant> {
rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now)
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
opts: &ReadOptions,
allow_coalescing: bool,
) -> disk::error::Result<FileInfo> {
if !allow_coalescing || !read_version_coalescing_enabled() {
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
if !matches!(disk.as_ref(), Disk::Remote(_)) {
record_read_version_coalescer_event("bypass_non_remote", 1);
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
let (tx, rx) = oneshot::channel();
let item = BatchReadVersionItem {
org_volume: org_bucket.to_string(),
volume: bucket.to_string(),
path: object.to_string(),
version_id: version_id.to_string(),
};
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
let lane = coalescer.lanes.entry(lane_key).or_default();
let schedule_delayed_flush = lane.is_empty();
lane.push(CoalescedReadVersionRequest { item, tx });
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
coalescer.lanes.remove(&lane_key)
} else if schedule_delayed_flush {
let disk = disk.clone();
let task_opts = *opts;
tokio::spawn(async move {
tokio::time::sleep(read_version_coalescing_delay()).await;
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
});
None
} else {
None
}
};
if let Some(pending) = pending {
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
let wait_started = batch_read_version_stage_timer();
let response = rx
.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")));
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started);
response
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
coalescer.lanes.remove(&lane_key).unwrap_or_default()
};
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
}
async fn flush_read_version_coalescer_pending(
lane_key: ReadVersionCoalescerKey,
disk: DiskStore,
opts: ReadOptions,
pending: Vec<CoalescedReadVersionRequest>,
) {
if pending.is_empty() {
return;
}
#[cfg(not(test))]
let _ = lane_key;
#[cfg(test)]
{
let mut observed_paths = HashSet::new();
for request in &pending {
if observed_paths.insert(request.item.path.as_str()) {
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
}
}
}
let mut senders = Vec::with_capacity(pending.len());
let mut items = Vec::with_capacity(pending.len());
for request in pending {
senders.push(request.tx);
items.push(request.item);
}
let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::<Vec<_>>();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
.await
{
Ok(result) => result,
Err(_) => Err(DiskError::Timeout),
};
match result {
Ok(responses) => {
let map_started = batch_read_version_stage_timer();
let results = map_batch_read_version_responses(&expected_items, responses);
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
}
Err(err) => {
let message = err.to_string();
for tx in senders {
let _ = tx.send(Err(DiskError::other(message.clone())));
}
}
}
}
fn map_batch_read_version_responses(
expected_items: &[ExpectedBatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
let mut seen = vec![false; expected_items.len()];
for response in responses {
let Some(expected) = expected_items.get(response.index) else {
continue;
};
let Some(slot) = results.get_mut(response.index) else {
continue;
};
if seen[response.index] {
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
continue;
}
seen[response.index] = true;
if response.path != expected.path || response.version_id != expected.version_id {
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
} else {
*slot = if response.success {
Ok(response.file_info)
} else {
Err(batch_read_version_response_error(response.error_code, response.error))
};
}
}
results
}
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
match DiskError::from_u32(error_code) {
Some(DiskError::Io(_)) | None => DiskError::other(error),
Some(error) => error,
}
}
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
bucket: &str,
object: &str,
total_disks: usize,
default_parity_count: usize,
) -> Vec<usize> {
let fallback_order = || (0..total_disks).collect::<Vec<_>>();
if default_parity_count == 0 || default_parity_count >= total_disks {
return fallback_order();
}
let data_blocks = total_disks - default_parity_count;
let distribution_key = metadata_distribution_key(bucket, object);
let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count)
.erasure
.distribution;
if distribution.len() != total_disks {
return fallback_order();
}
let mut order = Vec::with_capacity(total_disks);
for block_index in 1..=data_blocks {
let Some(disk_index) = distribution
.iter()
.position(|distributed_block| *distributed_block == block_index)
else {
return fallback_order();
};
order.push(disk_index);
}
order.extend(
distribution
.iter()
.enumerate()
.filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)),
);
order
}
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::{Mutex, RwLock, oneshot};
use tokio::task::JoinSet;
struct AbortOnDropJoinHandle<T>(tokio::task::JoinHandle<T>);
impl<T> Future for AbortOnDropJoinHandle<T> {
type Output = std::result::Result<T, tokio::task::JoinError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
Pin::new(&mut self.0).poll(cx)
}
}
impl<T> Drop for AbortOnDropJoinHandle<T> {
fn drop(&mut self) {
self.0.abort();
}
}
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
pub(crate) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
const DEFAULT_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: bool = true;
pub(in crate::set_disk) const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP: &str =
"RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP";
pub(in crate::set_disk) const SLOW_OBJECT_READ_LOG_THRESHOLD: Duration = Duration::from_secs(5);
pub(in crate::set_disk) const READ_REPAIR_HEAL_DEDUP_TTL: Duration = Duration::from_secs(60);
pub(in crate::set_disk) const READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES: usize = 4096;
pub(in crate::set_disk) static READ_REPAIR_HEAL_CACHE: OnceLock<RwLock<HashMap<ReadRepairHealCacheKey, Instant>>> =
OnceLock::new();
pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
Reader(Box<dyn AsyncRead + Unpin + Send + Sync>),
Fallback(GetCodecStreamingFallbackReason),
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
}
impl MultipartCodecStreamingReader {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
Self {
readers: VecDeque::from(readers),
}
}
}
impl AsyncRead for MultipartCodecStreamingReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if buf.remaining() == 0 {
return Poll::Ready(Ok(()));
}
loop {
let Some(reader) = self.readers.front_mut() else {
return Poll::Ready(Ok(()));
};
let filled_before = buf.filled().len();
match Pin::new(reader).poll_read(cx, buf) {
Poll::Ready(Ok(())) if buf.filled().len() == filled_before => {
self.readers.pop_front();
}
result => return result,
}
}
}
}
pub(in crate::set_disk) fn codec_streaming_reader_setup_fallback_reason(
missing_shards: usize,
) -> Option<GetCodecStreamingFallbackReason> {
(missing_shards > 0).then_some(GetCodecStreamingFallbackReason::ReadQuorumNotSafe)
}
#[derive(Clone, Copy, Debug)]
pub(in crate::set_disk) struct MetadataFanoutObservation {
pub(in crate::set_disk) outcome: &'static str,
pub(in crate::set_disk) elapsed: Duration,
pub(in crate::set_disk) valid: bool,
pub(in crate::set_disk) ignored: bool,
}
impl MetadataFanoutObservation {
pub(in crate::set_disk) fn from_file_info(file_info: &FileInfo, elapsed: Duration) -> Self {
if file_info_is_valid_for_metadata(file_info) {
Self {
outcome: GET_METADATA_RESPONSE_VALID,
elapsed,
valid: true,
ignored: false,
}
} else {
Self {
outcome: GET_METADATA_RESPONSE_ERROR,
elapsed,
valid: false,
ignored: false,
}
}
}
pub(in crate::set_disk) fn from_error(err: &DiskError, elapsed: Duration) -> Self {
Self {
outcome: classify_metadata_response_error(err),
elapsed,
valid: false,
ignored: is_metadata_fanout_ignored_error(err),
}
}
}
#[derive(Clone, Debug, Default)]
pub(in crate::set_disk) struct MetadataFanoutDiagnostics {
pub(in crate::set_disk) fanout_duration: Duration,
pub(in crate::set_disk) observations: Vec<MetadataFanoutObservation>,
}
impl MetadataFanoutDiagnostics {
pub(in crate::set_disk) fn new(fanout_duration: Duration, observations: Vec<MetadataFanoutObservation>) -> Self {
Self {
fanout_duration,
observations,
}
}
pub(in crate::set_disk) fn total_responses(&self) -> usize {
self.observations.len()
}
pub(in crate::set_disk) fn valid_responses(&self) -> usize {
self.observations.iter().filter(|observation| observation.valid).count()
}
pub(in crate::set_disk) fn ignored_responses(&self) -> usize {
self.observations.iter().filter(|observation| observation.ignored).count()
}
pub(in crate::set_disk) fn non_valid_responses(&self) -> usize {
self.total_responses().saturating_sub(self.valid_responses())
}
pub(in crate::set_disk) fn first_response_latency(&self) -> Option<Duration> {
self.observations.iter().map(|observation| observation.elapsed).min()
}
pub(in crate::set_disk) fn first_valid_response_latency(&self) -> Option<Duration> {
self.observations
.iter()
.filter(|observation| observation.valid)
.map(|observation| observation.elapsed)
.min()
}
pub(in crate::set_disk) fn slowest_response_latency(&self) -> Option<Duration> {
self.observations.iter().map(|observation| observation.elapsed).max()
}
pub(in crate::set_disk) fn quorum_candidate_latency(&self, read_quorum: usize) -> Option<Duration> {
if read_quorum == 0 {
return Some(Duration::ZERO);
}
let mut valid_latencies = self
.observations
.iter()
.filter(|observation| observation.valid)
.map(|observation| observation.elapsed)
.collect::<Vec<_>>();
valid_latencies.sort_unstable();
valid_latencies.get(read_quorum.saturating_sub(1)).copied()
}
pub(in crate::set_disk) fn record(&self, path: &'static str) {
rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64());
if let Some(latency) = self.first_response_latency() {
rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64());
}
if let Some(latency) = self.first_valid_response_latency() {
rustfs_io_metrics::record_get_object_first_valid_metadata_response_latency(path, latency.as_secs_f64());
}
if let Some(latency) = self.slowest_response_latency() {
rustfs_io_metrics::record_get_object_slowest_metadata_response_latency(path, latency.as_secs_f64());
}
rustfs_io_metrics::record_get_object_metadata_fanout_shape(
path,
self.total_responses(),
self.valid_responses(),
self.ignored_responses(),
self.non_valid_responses(),
);
for observation in &self.observations {
rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome);
}
}
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
}
}
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) enum MetadataCacheLookup {
Hit(Arc<GetObjectMetadataCacheEntry>),
Miss,
RejectedInsufficientQuorum,
}
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
bucket: &str,
object: &str,
candidate: &FileInfo,
parts_metadata: &[FileInfo],
disks: &[Option<DiskStore>],
) -> Option<&'static str> {
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
return Some(reason);
}
let Ok(erasure) = coding::Erasure::try_new_with_options(
candidate.erasure.data_blocks,
candidate.erasure.parity_blocks,
candidate.erasure.block_size,
candidate.uses_legacy_checksum,
) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
};
let data_files =
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
disks.get(index).is_some_and(Option::is_some)
}) {
Ok(data_files) => data_files,
Err(reason) => return Some(reason),
};
let Some(part) = candidate.parts.first() else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
};
let Ok(object_size) = usize::try_from(candidate.size) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
};
let checksum_info = candidate.erasure.get_checksum_info(part.number);
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let read_length = inline_erasure_shard_file_offset(
0,
object_size,
object_size,
candidate.erasure.block_size,
erasure.data_shards,
candidate.uses_legacy_checksum,
);
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
let Ok(mut readers) =
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
};
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
Some(body) if body.len() == object_size => None,
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
}
}
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
}
if candidate.is_compressed()
|| candidate
.metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
{
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED);
}
if candidate.is_remote() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
}
if candidate.deleted {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
}
if candidate.size <= 0 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}
if candidate.parts.len() != 1 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
}
if !candidate.has_valid_erasure_geometry() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
}
let Ok(object_size) = usize::try_from(candidate.size) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
};
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
}
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}
None
}
pub(in crate::set_disk) fn non_inline_data_read_candidate_is_safe(candidate: &FileInfo) -> bool {
if candidate.inline_data()
|| candidate.is_compressed()
|| candidate.is_remote()
|| candidate
.metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|| candidate.parts.len() != 1
{
return false;
}
candidate.has_valid_erasure_geometry()
}
pub(in crate::set_disk) fn late_materialization_candidate_is_safe(candidate: &FileInfo) -> bool {
non_inline_data_read_candidate_is_safe(candidate)
&& candidate.size > 512 * 1024
&& object_fits_single_block(candidate.size, candidate.erasure.block_size)
}
pub(in crate::set_disk) fn non_inline_data_read_early_stop_allowed(read_data: bool, bucket: &str, object: &str) -> bool {
read_data && is_get_metadata_non_inline_data_read_early_stop_enabled() && !codec_streaming_rollout_applies(bucket, object)
}
const NON_INLINE_SINGLE_PENDING_HEDGE_DELAY: Duration = Duration::from_millis(100);
fn data_read_inline_missing_shards_are_pending(
candidate: &FileInfo,
parts_metadata: &[FileInfo],
errors: &[Option<DiskError>],
disks: &[Option<DiskStore>],
fanout_order: &[usize],
scheduled_fanout_len: usize,
) -> bool {
let Ok(erasure) = coding::Erasure::try_new_with_options(
candidate.erasure.data_blocks,
candidate.erasure.parity_blocks,
candidate.erasure.block_size,
candidate.uses_legacy_checksum,
) else {
return false;
};
let distribution = &candidate.erasure.distribution;
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
let mut missing_pending_data_shards = 0usize;
for (disk_index, file_info) in parts_metadata.iter().enumerate() {
let Some(&block_index) = distribution.get(disk_index) else {
return false;
};
if block_index == 0 || block_index > erasure.data_shards {
continue;
}
if !disks.get(disk_index).is_some_and(Option::is_some) {
return false;
}
let data_slot = block_index - 1;
if file_info.name.is_empty() {
let scheduled_and_not_failed = fanout_order
.get(..scheduled_fanout_len)
.is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index))
&& errors.get(disk_index).is_some_and(Option::is_none);
if scheduled_and_not_failed {
data_shards_seen_or_pending[data_slot] = true;
missing_pending_data_shards = missing_pending_data_shards.saturating_add(1);
continue;
}
return false;
}
if file_info.erasure.index != block_index
|| !file_info.has_valid_erasure_geometry()
|| !metadata_early_stop_candidate_matches(file_info, candidate)
|| file_info.data.as_ref().is_none_or(|data| data.is_empty())
{
return false;
}
data_shards_seen_or_pending[data_slot] = true;
}
missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending)
}
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => GET_METADATA_RESPONSE_NOT_FOUND,
DiskError::FileVersionNotFound => GET_METADATA_RESPONSE_VERSION_NOT_FOUND,
DiskError::DiskNotFound => GET_METADATA_RESPONSE_DISK_NOT_FOUND,
DiskError::FileCorrupt | DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::OutdatedXLMeta => {
GET_METADATA_RESPONSE_CORRUPT
}
DiskError::Timeout => GET_METADATA_RESPONSE_TIMEOUT,
DiskError::FaultyDisk | DiskError::FaultyRemoteDisk => GET_METADATA_RESPONSE_IGNORED,
_ => GET_METADATA_RESPONSE_ERROR,
}
}
pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool {
let Some(err) = err else {
return false;
};
err.contains("file not found")
|| err.contains("No such file or directory")
|| err.contains("Specified part could not be found")
|| (err.starts_with("part.") && err.ends_with(" not found"))
}
pub(in crate::set_disk) fn resolve_read_part_from_responses(
bucket: &str,
part_meta_path: &str,
part_number: usize,
part_idx: usize,
expected_part_count: usize,
responses: &[Option<Vec<ObjectPartInfo>>],
read_quorum: usize,
) -> disk::error::Result<ObjectPartInfo> {
let mut part_quorum = HashMap::new();
let mut present_count = 0usize;
let mut missing_count = 0usize;
let mut transient_error_count = 0usize;
let mut mismatched_response_count = 0usize;
for response in responses.iter() {
let Some(parts) = response else {
transient_error_count += 1;
continue;
};
if parts.len() != expected_part_count {
mismatched_response_count += 1;
continue;
}
if !parts[part_idx].etag.is_empty() {
present_count += 1;
let part = &parts[part_idx];
let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
let (count, _) = part_quorum.entry(key).or_insert((0, part));
*count += 1;
continue;
}
if is_confirmed_missing_part_error(parts[part_idx].error.as_deref()) {
missing_count += 1;
} else {
transient_error_count += 1;
}
}
let max_part = part_quorum.values().max_by_key(|(count, _)| count);
let max_quorum = max_part.map_or(0, |(count, _)| *count).max(missing_count);
if let Some((count, part)) = max_part
&& *count >= read_quorum
{
return Ok((*part).clone());
}
if missing_count >= read_quorum {
return Ok(ObjectPartInfo {
number: part_number,
error: Some(format!("part.{part_number} not found")),
..Default::default()
});
}
if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
part_meta_path = %part_meta_path,
part_id = part_number,
read_quorum = read_quorum,
max_quorum = max_quorum,
disk_response_count = responses.len(),
present_count = present_count,
missing_count = missing_count,
transient_error_count = transient_error_count,
mismatched_response_count = mismatched_response_count,
"issue3031_read_parts_part_quorum"
);
}
Err(DiskError::ErasureReadQuorum)
}
pub(in crate::set_disk) fn shard_read_costs_for_disks(disks: &[Option<DiskStore>]) -> Vec<ShardReadCost> {
if disks.is_empty() {
return Vec::new();
}
let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs();
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref(), local_endpoint_hosts))
.collect()
}
pub(in crate::set_disk) fn shard_read_cost_for_disk(disk: Option<&DiskStore>, local_endpoint_hosts: &[String]) -> ShardReadCost {
match disk {
Some(disk) if disk.is_local() => ShardReadCost::Local,
Some(disk) => shard_read_cost_for_endpoint(false, &disk.host_name(), local_endpoint_hosts),
None => ShardReadCost::Unknown,
}
}
pub(in crate::set_disk) fn shard_read_cost_for_endpoint(
is_local: bool,
host_name: &str,
local_endpoint_hosts: &[String],
) -> ShardReadCost {
if is_local {
return ShardReadCost::Local;
}
if !host_name.is_empty() && local_endpoint_hosts.iter().any(|host| host == host_name) {
return ShardReadCost::SameNode;
}
ShardReadCost::Remote
}
pub(in crate::set_disk) fn local_endpoint_hosts_for_shard_costs() -> &'static [String] {
static LOCAL_ENDPOINT_HOSTS: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
if let Some(hosts) = LOCAL_ENDPOINT_HOSTS.get() {
return hosts;
}
let Some(endpoint_pools) = runtime_sources::endpoint_pools() else {
return &[];
};
let mut hosts = Vec::new();
for pool in endpoint_pools.as_ref() {
for endpoint in pool.endpoints.as_ref() {
if !endpoint.is_local {
continue;
}
let host = endpoint.host_port();
if !host.is_empty() && !hosts.contains(&host) {
hosts.push(host);
}
}
}
LOCAL_ENDPOINT_HOSTS.get_or_init(|| hosts)
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub(in crate::set_disk) struct ReadRepairHealCacheKey {
pub(in crate::set_disk) bucket: String,
pub(in crate::set_disk) object: String,
pub(in crate::set_disk) version_id: Option<String>,
pub(in crate::set_disk) pool_index: usize,
pub(in crate::set_disk) set_index: usize,
}
impl ReadRepairHealCacheKey {
pub(in crate::set_disk) fn new(
bucket: &str,
object: &str,
version_id: Option<&str>,
pool_index: usize,
set_index: usize,
) -> Self {
Self {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: version_id.filter(|value| !value.is_empty()).map(str::to_string),
pool_index,
set_index,
}
}
}
pub(in crate::set_disk) fn resolved_read_repair_version_id(fi: &FileInfo, requested_version_id: Option<&str>) -> Option<String> {
fi.version_id
.as_ref()
.map(ToString::to_string)
.or_else(|| requested_version_id.filter(|value| !value.is_empty()).map(str::to_string))
}
pub(in crate::set_disk) async fn reserve_read_repair_heal(
bucket: &str,
object: &str,
version_id: Option<&str>,
pool_index: usize,
set_index: usize,
) -> Option<ReadRepairHealCacheKey> {
let key = ReadRepairHealCacheKey::new(bucket, object, version_id, pool_index, set_index);
let now = Instant::now();
let cache = READ_REPAIR_HEAL_CACHE.get_or_init(|| RwLock::new(HashMap::new()));
{
let cache = cache.read().await;
if cache
.get(&key)
.is_some_and(|seen_at| now.saturating_duration_since(*seen_at) <= READ_REPAIR_HEAL_DEDUP_TTL)
{
return None;
}
}
let mut cache = cache.write().await;
if cache
.get(&key)
.is_some_and(|seen_at| now.saturating_duration_since(*seen_at) <= READ_REPAIR_HEAL_DEDUP_TTL)
{
return None;
}
if cache.len() >= READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES {
cache.retain(|_, seen_at| now.saturating_duration_since(*seen_at) <= READ_REPAIR_HEAL_DEDUP_TTL);
}
if cache.len() >= READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES
&& let Some(oldest_key) = cache.iter().min_by_key(|(_, seen_at)| **seen_at).map(|(key, _)| key.clone())
{
cache.remove(&oldest_key);
}
cache.insert(key.clone(), now);
Some(key)
}
pub(in crate::set_disk) async fn release_read_repair_heal_reservation(key: &ReadRepairHealCacheKey) {
if let Some(cache) = READ_REPAIR_HEAL_CACHE.get() {
cache.write().await.remove(key);
}
}
pub(in crate::set_disk) fn record_read_repair_dedup(reason: &'static str) {
counter!("rustfs_heal_read_repair_dedup_total", "reason" => reason).increment(1);
}
#[derive(Debug)]
pub(in crate::set_disk) enum ReadRepairAdmissionOutcome {
Response(HealAdmissionResult),
Failed(String),
}
pub(in crate::set_disk) type ReadRepairAdmissionFuture = Pin<Box<dyn Future<Output = ReadRepairAdmissionOutcome> + Send>>;
pub(in crate::set_disk) type ReadRepairAdmissionSubmitter =
fn(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture;
pub(in crate::set_disk) struct ReadRepairHealSubmission<'a> {
pub(in crate::set_disk) bucket: &'a str,
pub(in crate::set_disk) object: &'a str,
pub(in crate::set_disk) version_id: Option<&'a str>,
pub(in crate::set_disk) pool_index: usize,
pub(in crate::set_disk) set_index: usize,
pub(in crate::set_disk) part_number: Option<usize>,
pub(in crate::set_disk) reason: &'static str,
pub(in crate::set_disk) mrf_intent: Option<(rustfs_common::mrf_channel::MrfKind, Option<uuid::Uuid>)>,
}
pub(in crate::set_disk) fn send_read_repair_heal_request(
request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
) -> ReadRepairAdmissionFuture {
Box::pin(async {
match send_heal_request_with_admission(request).await {
Ok(result) => ReadRepairAdmissionOutcome::Response(result),
Err(err) => ReadRepairAdmissionOutcome::Failed(err),
}
})
}
pub(in crate::set_disk) async fn submit_read_repair_heal(
bucket: &str,
object: &str,
version_id: Option<&str>,
pool_index: usize,
set_index: usize,
part_number: Option<usize>,
reason: &'static str,
) {
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket,
object,
version_id,
pool_index,
set_index,
part_number,
reason,
mrf_intent: None,
},
send_read_repair_heal_request,
)
.await;
}
pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
submission: ReadRepairHealSubmission<'_>,
submitter: ReadRepairAdmissionSubmitter,
) {
let ReadRepairHealSubmission {
bucket,
object,
version_id,
pool_index,
set_index,
part_number,
reason,
mrf_intent,
} = submission;
if let Some((kind, version_uuid)) = mrf_intent
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
{
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
let ingress = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
debug!(
event = EVENT_SET_DISK_READ,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "mrf_ingress",
mrf_kind = ?kind,
bucket,
object,
version_id,
pool_index,
set_index,
result = ?ingress,
"Read-repair MRF ingress result"
);
}
let Some(dedup_key) = reserve_read_repair_heal(bucket, object, version_id, pool_index, set_index).await else {
record_read_repair_dedup("duplicate");
debug!(
bucket,
object, part_number, pool_index, set_index, reason, "Skipped duplicate read-repair heal request"
);
return;
};
let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(HealChannelPriority::Low),
Some(pool_index),
Some(set_index),
);
request.source = HealRequestSource::ReadRepair;
request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
request.recreate_missing = Some(true);
let request_id = request.id.clone();
let bucket = bucket.to_string();
let object = object.to_string();
tokio::spawn(async move {
match submitter(request).await {
ReadRepairAdmissionOutcome::Response(result) if result.is_admitted() => {
debug!(
bucket,
object,
part_number,
pool_index,
set_index,
request_id,
reason,
admission = result.result_label(),
"Read-repair heal request admitted"
);
}
ReadRepairAdmissionOutcome::Response(result) => {
release_read_repair_heal_reservation(&dedup_key).await;
debug!(
bucket,
object,
part_number,
pool_index,
set_index,
request_id,
reason,
admission = result.result_label(),
drop_reason = result.reason_label(),
"Read-repair heal request not admitted"
);
}
ReadRepairAdmissionOutcome::Failed(err) => {
release_read_repair_heal_reservation(&dedup_key).await;
debug!(
bucket,
object,
part_number,
pool_index,
set_index,
request_id,
reason,
error = %err,
"Read-repair heal request could not be submitted"
);
}
}
});
}
pub(in crate::set_disk) type ObjectBitrotReader = BitrotReader<ShardReader>;
pub(in crate::set_disk) type DeferredReaderReopener = crate::erasure::coding::decode::DeferredReaderReopener<ShardReader>;
pub(in crate::set_disk) type BitrotReaderTask<'a> =
Pin<Box<dyn Future<Output = (usize, std::result::Result<Option<ObjectBitrotReader>, DiskError>)> + Send + 'a>>;
pub(in crate::set_disk) const DIRECT_MEMORY_BITROT_READER_STAGE_METRICS: BitrotReaderStageMetrics = BitrotReaderStageMetrics {
path: GET_OBJECT_PATH_DIRECT_MEMORY,
reader_construction_stage: GET_STAGE_READER_TASK_READER_CONSTRUCTION,
file_open_stage: GET_STAGE_READER_TASK_FILE_OPEN,
bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT,
};
pub(in crate::set_disk) struct BitrotReaderSetup {
pub(in crate::set_disk) readers: Vec<Option<ObjectBitrotReader>>,
pub(in crate::set_disk) deferred_stripe_handles: Vec<Option<DeferredReaderStripeHandle>>,
pub(in crate::set_disk) deferred_reopeners: Vec<Option<DeferredReaderReopener>>,
pub(in crate::set_disk) errors: Vec<Option<DiskError>>,
pub(in crate::set_disk) scheduled: Vec<bool>,
pub(in crate::set_disk) attempted: Vec<bool>,
pub(in crate::set_disk) ready: Vec<bool>,
pub(in crate::set_disk) scheduled_count: usize,
pub(in crate::set_disk) attempted_count: usize,
pub(in crate::set_disk) ready_count: usize,
pub(in crate::set_disk) failed_count: usize,
pub(in crate::set_disk) deferred_count: usize,
}
#[derive(Clone, Copy)]
pub(in crate::set_disk) struct BitrotReaderSetupAttribution {
pub(in crate::set_disk) path: &'static str,
pub(in crate::set_disk) object_class: &'static str,
pub(in crate::set_disk) size_bucket: &'static str,
}
#[derive(Clone, Copy)]
pub(in crate::set_disk) enum BitrotReaderSetupMode {
ReadQuorum,
VerifyReconstruction,
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(in crate::set_disk) enum BitrotReaderSetupStrategy {
AllShards,
DataBlocksFirst,
DataBlocksOnly,
}
impl BitrotReaderSetupMode {
pub(in crate::set_disk) fn as_str(self) -> &'static str {
match self {
BitrotReaderSetupMode::ReadQuorum => "read_quorum",
BitrotReaderSetupMode::VerifyReconstruction => "verify_reconstruction",
}
}
}
impl BitrotReaderSetupStrategy {
pub(in crate::set_disk) fn as_str(self) -> &'static str {
match self {
BitrotReaderSetupStrategy::AllShards => "all_shards",
BitrotReaderSetupStrategy::DataBlocksFirst => "data_blocks_first",
BitrotReaderSetupStrategy::DataBlocksOnly => "data_blocks_only",
}
}
}
pub(in crate::set_disk) fn get_bitrot_reader_setup_strategy(
mode: BitrotReaderSetupMode,
prefer_data_blocks_first: bool,
) -> BitrotReaderSetupStrategy {
if matches!(
crate::set_disk::get_object_read_policy(),
crate::set_disk::GetObjectReadPolicy::CopySource
) {
return BitrotReaderSetupStrategy::DataBlocksFirst;
}
match mode {
BitrotReaderSetupMode::ReadQuorum
if prefer_data_blocks_first
|| rustfs_utils::get_env_bool(
ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP,
DEFAULT_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP,
) =>
{
BitrotReaderSetupStrategy::DataBlocksFirst
}
BitrotReaderSetupMode::VerifyReconstruction
if prefer_data_blocks_first
|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, false) =>
{
BitrotReaderSetupStrategy::DataBlocksFirst
}
_ => BitrotReaderSetupStrategy::AllShards,
}
}
impl BitrotReaderSetup {
pub(in crate::set_disk) fn bind_integrity(
&mut self,
expected: Option<&rustfs_filemeta::shard_integrity::PartIntegrity>,
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
first_stripe: usize,
) -> std::io::Result<()> {
use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier};
let Some(expected) = expected else { return Ok(()) };
let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?;
for (index, reader) in self.readers.iter_mut().enumerate() {
if let Some(reader) = reader {
let advanced = self.deferred_stripe_handles[index]
.as_ref()
.map(DeferredReaderStripeHandle::integrity_position);
reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first_stripe, advanced)?)?;
}
if let Some(reopen) = self.deferred_reopeners[index].take() {
let proof = Arc::clone(&proof);
self.deferred_reopeners[index] = Some(Arc::new(move |stripe| {
let mut reader = reopen(stripe)?;
let first = first_stripe.checked_add(stripe)?;
reader
.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first, None).ok()?)
.ok()?;
Some(reader)
}));
}
}
Ok(())
}
pub(in crate::set_disk) fn new(shards: usize) -> Self {
Self {
readers: (0..shards).map(|_| None).collect(),
deferred_stripe_handles: (0..shards).map(|_| None).collect(),
deferred_reopeners: (0..shards).map(|_| None).collect(),
errors: vec![Some(DiskError::DiskNotFound); shards],
scheduled: vec![false; shards],
attempted: vec![false; shards],
ready: vec![false; shards],
scheduled_count: 0,
attempted_count: 0,
ready_count: 0,
failed_count: 0,
deferred_count: 0,
}
}
pub(in crate::set_disk) fn scheduled_shards(&self) -> usize {
self.scheduled_count
}
pub(in crate::set_disk) fn attempted_shards(&self) -> usize {
self.attempted_count
}
pub(in crate::set_disk) fn pending_scheduled_shards(&self) -> usize {
self.scheduled_count.saturating_sub(self.attempted_count)
}
pub(in crate::set_disk) fn available_shards(&self) -> usize {
self.ready_count
}
pub(in crate::set_disk) fn available_data_shards(&self, data_shards: usize) -> usize {
self.ready.iter().take(data_shards).filter(|ready| **ready).count()
}
pub(in crate::set_disk) fn completed_failed_shards(&self) -> usize {
self.failed_count
}
pub(in crate::set_disk) fn data_shards_attempted(&self, data_shards: usize) -> bool {
self.attempted.iter().take(data_shards).all(|attempted| *attempted)
}
pub(in crate::set_disk) fn reconstruction_verification_target(&self, data_shards: usize, parity_shards: usize) -> usize {
let missing_data_sources = data_shards.saturating_sub(self.available_data_shards(data_shards));
if missing_data_sources > 0 && missing_data_sources < parity_shards {
data_shards.saturating_add(1).min(data_shards.saturating_add(parity_shards))
} else {
data_shards
}
}
pub(in crate::set_disk) fn has_setup_quorum(
&self,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
) -> bool {
self.available_shards() >= self.setup_target(data_shards, parity_shards, mode)
}
pub(in crate::set_disk) fn setup_target(
&self,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
) -> usize {
match mode {
BitrotReaderSetupMode::ReadQuorum => data_shards,
BitrotReaderSetupMode::VerifyReconstruction => self.reconstruction_verification_target(data_shards, parity_shards),
}
}
pub(in crate::set_disk) fn scheduling_target(
&self,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
) -> usize {
match mode {
BitrotReaderSetupMode::ReadQuorum => data_shards,
BitrotReaderSetupMode::VerifyReconstruction if !self.data_shards_attempted(data_shards) => data_shards,
BitrotReaderSetupMode::VerifyReconstruction => self.reconstruction_verification_target(data_shards, parity_shards),
}
}
pub(in crate::set_disk) fn deferred_shards(&self) -> usize {
self.deferred_count
}
pub(in crate::set_disk) fn mark_scheduled(&mut self, idx: usize) -> bool {
if self.scheduled[idx] {
return false;
}
self.scheduled[idx] = true;
self.scheduled_count = self.scheduled_count.saturating_add(1);
true
}
pub(in crate::set_disk) fn apply_reader_result(
&mut self,
idx: usize,
result: std::result::Result<Option<ObjectBitrotReader>, DiskError>,
) {
self.attempted[idx] = true;
self.attempted_count = self.attempted_count.saturating_add(1);
match result {
Ok(Some(reader)) => {
self.readers[idx] = Some(reader);
self.errors[idx] = None;
self.ready[idx] = true;
self.ready_count = self.ready_count.saturating_add(1);
}
Ok(None) => {
self.readers[idx] = None;
self.errors[idx] = Some(DiskError::DiskNotFound);
self.ready[idx] = false;
self.failed_count = self.failed_count.saturating_add(1);
}
Err(e) => {
self.readers[idx] = None;
self.errors[idx] = Some(e);
self.ready[idx] = false;
self.failed_count = self.failed_count.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn retain_deferred_reader(
&mut self,
idx: usize,
reader: ObjectBitrotReader,
stripe_handle: DeferredReaderStripeHandle,
) {
self.readers[idx] = Some(reader);
self.deferred_stripe_handles[idx] = Some(stripe_handle);
self.errors[idx] = None;
self.deferred_count = self.deferred_count.saturating_add(1);
}
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) fn schedule_bitrot_reader_task<'a>(
reader_tasks: &mut FuturesUnordered<BitrotReaderTask<'a>>,
setup: &mut BitrotReaderSetup,
idx: usize,
files: &'a [FileInfo],
disks: &'a [Option<DiskStore>],
bucket: &'a str,
object: &'a str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
stage_metrics: Option<BitrotReaderStageMetrics>,
) {
if idx >= disks.len() || !setup.mark_scheduled(idx) {
return;
}
let inline_data = files[idx].data.clone();
let data_dir = files[idx].data_dir.unwrap_or_default();
let disk = disks[idx].as_ref();
let path = format!("{object}/{data_dir}/part.{part_number}");
reader_tasks.push(Box::pin(async move {
let result = create_bitrot_reader_from_bytes_with_stage_metrics(
inline_data,
disk,
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
stage_metrics,
)
.await;
(idx, result)
}));
}
pub(in crate::set_disk) fn next_unscheduled_reader_index(
setup: &BitrotReaderSetup,
total_shards: usize,
data_shards: usize,
) -> Option<usize> {
(data_shards..total_shards)
.chain(0..data_shards.min(total_shards))
.find(|idx| !setup.scheduled[*idx])
}
#[allow(clippy::too_many_arguments)]
fn deferred_reader_reopener(
inline_data: Option<Bytes>,
disk: Option<DiskStore>,
bucket: &str,
path: &str,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
) -> DeferredReaderReopener {
let bucket = bucket.to_owned();
let path = path.to_owned();
Arc::new(move |stripe_index| {
let (reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
inline_data.clone(),
disk.clone(),
&bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
);
handle.advance_stripes(stripe_index).then_some(reader)
})
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
setup: &mut BitrotReaderSetup,
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
) {
if !setup.has_setup_quorum(data_shards, parity_shards, mode) {
return;
}
let demand_bound_lockstep = crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
let preserve_hedged_readers = !demand_bound_lockstep && read_length > shard_size;
for idx in 0..disks.len() {
if setup.attempted[idx] && (!preserve_hedged_readers || setup.readers[idx].is_none()) {
continue;
}
if files[idx].data.is_none() && disks[idx].is_none() {
continue;
}
let inline_data = files[idx].data.clone();
let disk = disks[idx].clone();
let data_dir = files[idx].data_dir.unwrap_or_default();
let path = format!("{object}/{data_dir}/part.{part_number}");
let reopener = (demand_bound_lockstep || preserve_hedged_readers).then(|| {
deferred_reader_reopener(
inline_data.clone(),
disk.clone(),
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
)
});
setup.deferred_reopeners[idx] = reopener;
if setup.attempted[idx] {
continue;
}
let (reader, stripe_handle) = create_deferred_bitrot_reader_with_stripe_handle(
inline_data,
disk,
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
);
setup.retain_deferred_reader(idx, reader, stripe_handle);
}
if !demand_bound_lockstep {
return;
}
for idx in data_shards..disks.len() {
if setup.readers[idx].is_none() || setup.deferred_stripe_handles[idx].is_some() {
continue;
}
let inline_data = files[idx].data.clone();
let disk = disks[idx].clone();
let data_dir = files[idx].data_dir.unwrap_or_default();
let path = format!("{object}/{data_dir}/part.{part_number}");
let reopener = demand_bound_lockstep.then(|| {
deferred_reader_reopener(
inline_data.clone(),
disk.clone(),
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
)
});
let (reader, stripe_handle) = create_deferred_bitrot_reader_with_stripe_handle(
inline_data,
disk,
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
);
setup.readers[idx] = Some(reader);
setup.deferred_stripe_handles[idx] = Some(stripe_handle);
setup.deferred_reopeners[idx] = reopener;
}
}
pub(in crate::set_disk) fn record_bitrot_reader_setup_fanout(
strategy: BitrotReaderSetupStrategy,
mode: BitrotReaderSetupMode,
setup: &BitrotReaderSetup,
attribution: Option<BitrotReaderSetupAttribution>,
) {
let strategy = strategy.as_str();
let mode = mode.as_str();
let scheduled = setup.scheduled_shards();
let attempted = setup.attempted_shards();
let ready = setup.available_shards();
let failed = setup.completed_failed_shards();
let deferred = setup.deferred_shards();
rustfs_io_metrics::record_get_object_reader_setup_fanout(strategy, mode, scheduled, attempted, ready, failed, deferred);
if let Some(attribution) = attribution {
rustfs_io_metrics::record_get_object_reader_setup_fanout_by_size(
attribution.path,
strategy,
mode,
attribution.object_class,
attribution.size_bucket,
scheduled,
attempted,
ready,
failed,
deferred,
);
}
}
pub(in crate::set_disk) fn record_bitrot_reader_setup_strategy(
strategy: BitrotReaderSetupStrategy,
mode: BitrotReaderSetupMode,
attribution: Option<BitrotReaderSetupAttribution>,
) {
let strategy = strategy.as_str();
let mode = mode.as_str();
rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy, mode);
if let Some(attribution) = attribution {
rustfs_io_metrics::record_get_object_reader_setup_strategy_by_size(
attribution.path,
strategy,
mode,
attribution.object_class,
attribution.size_bucket,
);
}
}
#[cfg(unix)]
#[allow(clippy::too_many_arguments)]
async fn try_create_bitrot_readers_via_batch_pread(
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
) -> Option<BitrotReaderSetup> {
use crate::disk::local::batch_shard_pread;
use std::io::Cursor;
let (adj_off, adj_len) = adjust_shard_read_params(read_offset, read_length, shard_size, &checksum_algo);
if adj_len > object_mmap_read_max_length() {
return None;
}
let mut batch_items: Vec<(usize, std::path::PathBuf, usize, usize)> = Vec::new();
for (idx, disk_op) in disks.iter().enumerate() {
if files.get(idx).is_some_and(|fi| fi.data.is_some()) {
return None;
}
if let Some(disk) = disk_op.as_ref() {
let data_dir = files[idx].data_dir.unwrap_or_default();
let path_str = format!("{object}/{data_dir}/part.{part_number}");
match disk.get_object_path_for_io_if_local(bucket, &path_str) {
Some(Ok(p)) => batch_items.push((idx, p, adj_off, adj_len)),
_ => return None,
}
}
}
if batch_items.is_empty() {
return None;
}
let requests: Vec<_> = batch_items.iter().map(|(_, p, off, len)| (p.clone(), *off, *len)).collect();
let batch_results = batch_shard_pread(requests).await;
let mut setup = BitrotReaderSetup::new(disks.len());
for (i, (idx, _, _, _)) in batch_items.iter().enumerate() {
setup.mark_scheduled(*idx);
match &batch_results[i] {
Ok(bytes) => {
let reader = BitrotReader::new(
ShardReader::InMemory(Cursor::new(bytes.clone())),
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
);
setup.apply_reader_result(*idx, Ok(Some(reader)));
}
Err(e) => {
setup.apply_reader_result(*idx, Err(e.clone()));
}
}
}
for (idx, disk_op) in disks.iter().enumerate() {
if setup.scheduled[idx] {
continue;
}
setup.mark_scheduled(idx);
if disk_op.is_none() {
setup.apply_reader_result(idx, Ok(None));
}
}
Some(setup)
}
#[cfg(not(unix))]
#[allow(clippy::too_many_arguments)]
async fn try_create_bitrot_readers_via_batch_pread(
_files: &[FileInfo],
_disks: &[Option<DiskStore>],
_bucket: &str,
_object: &str,
_part_number: usize,
_read_offset: usize,
_read_length: usize,
_shard_size: usize,
_checksum_algo: HashAlgorithm,
_skip_verify_bitrot: bool,
) -> Option<BitrotReaderSetup> {
None
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
stage_metrics: Option<BitrotReaderStageMetrics>,
attribution: Option<BitrotReaderSetupAttribution>,
) -> BitrotReaderSetup {
let strategy = BitrotReaderSetupStrategy::AllShards;
let mut setup = BitrotReaderSetup::new(disks.len());
let mut reader_tasks = FuturesUnordered::new();
let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
record_bitrot_reader_setup_strategy(strategy, mode, attribution);
let schedule_stage_start = stage_metrics.map(|_| Instant::now());
for (idx, disk_op) in disks.iter().enumerate() {
setup.mark_scheduled(idx);
let inline_data = files[idx].data.clone();
let data_dir = files[idx].data_dir.unwrap_or_default();
let disk = disk_op.as_ref();
let path = format!("{object}/{data_dir}/part.{part_number}");
let checksum_algo = checksum_algo.clone();
reader_tasks.push(async move {
let result = create_bitrot_reader_from_bytes_with_stage_metrics(
inline_data,
disk,
bucket,
&path,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
stage_metrics,
)
.await;
(idx, result)
});
}
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start);
}
let wait_quorum_stage_start = stage_metrics.map(|_| Instant::now());
while let Some((idx, result)) = reader_tasks.next().await {
setup.apply_reader_result(idx, result);
if setup.has_setup_quorum(data_shards, parity_shards, mode) {
break;
}
}
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_WAIT_QUORUM, wait_quorum_stage_start);
}
fill_deferred_bitrot_readers(
&mut setup,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
data_shards,
parity_shards,
mode,
);
let drop_pending_stage_start = stage_metrics.map(|_| Instant::now());
drop(reader_tasks);
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start);
}
record_bitrot_reader_setup_fanout(strategy, mode, &setup, attribution);
setup
}
#[allow(clippy::too_many_arguments)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
stage_metrics: Option<BitrotReaderStageMetrics>,
attribution: Option<BitrotReaderSetupAttribution>,
) -> BitrotReaderSetup {
create_bitrot_readers_until_quorum_with_preference(
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
data_shards,
parity_shards,
mode,
false,
stage_metrics,
attribution,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_with_preference(
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
data_shards: usize,
parity_shards: usize,
mode: BitrotReaderSetupMode,
prefer_data_blocks_first: bool,
stage_metrics: Option<BitrotReaderStageMetrics>,
attribution: Option<BitrotReaderSetupAttribution>,
) -> BitrotReaderSetup {
let strategy = get_bitrot_reader_setup_strategy(mode, prefer_data_blocks_first);
if use_mmap_read
&& !matches!(
crate::set_disk::get_object_read_policy(),
crate::set_disk::GetObjectReadPolicy::CopySource
)
&& let Some(mut setup) = try_create_bitrot_readers_via_batch_pread(
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
)
.await
{
record_bitrot_reader_setup_strategy(strategy, mode, attribution);
fill_deferred_bitrot_readers(
&mut setup,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
data_shards,
parity_shards,
mode,
);
record_bitrot_reader_setup_fanout(strategy, mode, &setup, attribution);
return setup;
}
if strategy == BitrotReaderSetupStrategy::AllShards {
return create_bitrot_readers_until_quorum_all_shards(
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
data_shards,
parity_shards,
mode,
stage_metrics,
attribution,
)
.await;
}
let mut setup = BitrotReaderSetup::new(disks.len());
let mut reader_tasks: FuturesUnordered<BitrotReaderTask<'_>> = FuturesUnordered::new();
let total_shards = disks.len();
let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
record_bitrot_reader_setup_strategy(strategy, mode, attribution);
let schedule_stage_start = stage_metrics.map(|_| Instant::now());
let initial_target = setup.setup_target(data_shards, parity_shards, mode);
for idx in 0..initial_target.min(data_shards).min(total_shards) {
schedule_bitrot_reader_task(
&mut reader_tasks,
&mut setup,
idx,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
stage_metrics,
);
}
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start);
}
let wait_quorum_stage_start = stage_metrics.map(|_| Instant::now());
while let Some((idx, result)) = reader_tasks.next().await {
setup.apply_reader_result(idx, result);
if setup.has_setup_quorum(data_shards, parity_shards, mode) {
break;
}
let target = setup.scheduling_target(data_shards, parity_shards, mode);
while setup.available_shards().saturating_add(setup.pending_scheduled_shards()) < target {
let Some(next_idx) = next_unscheduled_reader_index(&setup, total_shards, data_shards) else {
break;
};
schedule_bitrot_reader_task(
&mut reader_tasks,
&mut setup,
next_idx,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
stage_metrics,
);
}
}
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_WAIT_QUORUM, wait_quorum_stage_start);
}
fill_deferred_bitrot_readers(
&mut setup,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo,
skip_verify_bitrot,
use_mmap_read,
data_shards,
parity_shards,
mode,
);
let drop_pending_stage_start = stage_metrics.map(|_| Instant::now());
drop(reader_tasks);
if let Some(stage_metrics) = stage_metrics {
record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start);
}
record_bitrot_reader_setup_fanout(strategy, mode, &setup, attribution);
setup
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
part_number: usize,
read_offset: usize,
read_length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify_bitrot: bool,
use_mmap_read: bool,
data_shards: usize,
) -> BitrotReaderSetup {
let strategy = BitrotReaderSetupStrategy::DataBlocksOnly;
let total_shards = disks.len().min(files.len());
let mut setup = BitrotReaderSetup::new(total_shards);
let mut reader_tasks: FuturesUnordered<BitrotReaderTask<'_>> = FuturesUnordered::new();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let reader_stage_metrics = stage_metrics_enabled.then_some(DIRECT_MEMORY_BITROT_READER_STAGE_METRICS);
rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), BitrotReaderSetupMode::ReadQuorum.as_str());
let schedule_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
for idx in 0..data_shards.min(total_shards) {
schedule_bitrot_reader_task(
&mut reader_tasks,
&mut setup,
idx,
files,
disks,
bucket,
object,
part_number,
read_offset,
read_length,
shard_size,
checksum_algo.clone(),
skip_verify_bitrot,
use_mmap_read,
reader_stage_metrics,
);
}
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_DIRECT_MEMORY, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start);
let wait_quorum_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
while let Some((idx, result)) = reader_tasks.next().await {
setup.apply_reader_result(idx, result);
if setup.available_data_shards(data_shards) >= data_shards {
break;
}
}
record_get_stage_duration_if_enabled(
GET_OBJECT_PATH_DIRECT_MEMORY,
GET_STAGE_READER_SETUP_WAIT_QUORUM,
wait_quorum_stage_start,
);
let drop_pending_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
drop(reader_tasks);
record_get_stage_duration_if_enabled(
GET_OBJECT_PATH_DIRECT_MEMORY,
GET_STAGE_READER_SETUP_DROP_PENDING,
drop_pending_stage_start,
);
record_bitrot_reader_setup_fanout(strategy, BitrotReaderSetupMode::ReadQuorum, &setup, None);
setup
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
tasks: Vec<F>,
read_quorum: usize,
) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
where
F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
{
let mut responses = vec![None; tasks.len()];
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
let mut successful_responses = 0usize;
let mut pending = tasks.len();
let mut join_set = JoinSet::new();
for (index, task) in tasks.into_iter().enumerate() {
join_set.spawn(async move { (index, task.await) });
}
while let Some(join_result) = join_set.join_next().await {
pending = pending.saturating_sub(1);
match join_result {
Ok((index, Ok(resp))) => {
responses[index] = Some(resp);
errors[index] = None;
successful_responses += 1;
}
Ok((index, Err(err))) => {
errors[index] = Some(err);
}
Err(_) => {}
}
if successful_responses + pending < read_quorum {
return Err(());
}
}
Ok((responses, errors))
}
pub(in crate::set_disk) async fn collect_read_parts_results<F>(
tasks: Vec<F>,
read_quorum: usize,
) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
where
F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
{
let mut responses = vec![None; tasks.len()];
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
let mut successful_responses = 0usize;
let mut pending = tasks.len();
let mut join_set = JoinSet::new();
for (index, task) in tasks.into_iter().enumerate() {
join_set.spawn(async move { (index, task.await) });
}
while let Some(join_result) = join_set.join_next().await {
pending = pending.saturating_sub(1);
match join_result {
Ok((index, Ok(resp))) => {
responses[index] = Some(resp);
errors[index] = None;
successful_responses += 1;
}
Ok((index, Err(err))) => {
errors[index] = Some(err);
}
Err(_) => {}
}
if successful_responses + pending < read_quorum {
return Err(());
}
}
Ok((responses, errors))
}
pub(in crate::set_disk) fn should_allow_metadata_early_stop(
read_data: bool,
version_id: &str,
healing: bool,
incl_free_versions: bool,
) -> bool {
if read_data && !is_get_metadata_data_read_early_stop_enabled() {
return false;
}
(is_get_metadata_early_stop_enabled() && version_id.is_empty() && !healing && !incl_free_versions)
|| (is_version_early_stop_enabled() && !version_id.is_empty() && !healing && !incl_free_versions)
}
pub(in crate::set_disk) fn metadata_early_stop_permitted(
caller_allows_early_stop: bool,
observe: bool,
read_data: bool,
version_id: &str,
healing: bool,
incl_free_versions: bool,
) -> bool {
caller_allows_early_stop && observe && should_allow_metadata_early_stop(read_data, version_id, healing, incl_free_versions)
}
impl SetDisks {
pub(in crate::set_disk) async fn read_parts(
disks: &[Option<DiskStore>],
bucket: &str,
part_meta_paths: &[String],
part_numbers: &[usize],
read_quorum: usize,
) -> disk::error::Result<Vec<ObjectPartInfo>> {
let bucket: Arc<str> = Arc::from(bucket);
let part_meta_paths: Arc<[String]> = Arc::from(part_meta_paths);
let tasks: Vec<_> = disks
.iter()
.map(|disk| {
let disk = disk.clone();
let bucket = Arc::clone(&bucket);
let part_meta_paths = Arc::clone(&part_meta_paths);
async move {
if let Some(disk) = disk {
disk.read_parts(&bucket, &part_meta_paths).await
} else {
Err(DiskError::DiskNotFound)
}
}
})
.collect();
let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
Ok(collected) => collected,
Err(()) => return Err(DiskError::ErasureReadQuorum),
};
if let Some(err) = reduce_read_quorum_errs(&collected_errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
return Err(err);
}
let mut ret = vec![ObjectPartInfo::default(); part_meta_paths.len()];
for (part_idx, part_info) in part_meta_paths.iter().enumerate() {
ret[part_idx] = resolve_read_part_from_responses(
&bucket,
part_info,
part_numbers[part_idx],
part_idx,
part_meta_paths.len(),
&responses,
read_quorum,
)?;
}
Ok(ret)
}
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(level = "debug", skip(disks))]
pub(in crate::set_disk) async fn read_all_fileinfo(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
healing: bool,
incl_free_versions: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>)> {
let (ress, errors, _) = Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
healing,
incl_free_versions,
false,
true,
0,
false,
)
.await?;
Ok((ress, errors))
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
healing: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
healing,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
false,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
false,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
true,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn read_all_fileinfo_inner(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
healing: bool,
incl_free_versions: bool,
observe: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let early_stop_enabled =
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
let allow_early_stop =
metadata_early_stop_permitted(caller_allows_early_stop, observe, read_data, version_id, healing, incl_free_versions);
if allow_early_stop {
return Self::read_all_fileinfo_early_stop(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
healing,
incl_free_versions,
non_inline_data_read_early_stop_allowed(read_data, bucket, object),
default_parity_count,
allow_coalescing,
)
.await;
}
if early_stop_enabled {
let metrics_path = metadata_metrics_path(bucket);
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(
metrics_path,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
}
Self::read_all_fileinfo_full_wait(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
healing,
incl_free_versions,
observe,
allow_coalescing,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn read_all_fileinfo_full_wait(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
healing: bool,
incl_free_versions: bool,
observe: bool,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = observe.then(Instant::now);
let mut ress = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
let mut observations = observe.then(|| Vec::with_capacity(disks.len()));
let scheduled_count = disks.len();
let opts = ReadOptions {
incl_free_versions,
read_data,
healing,
};
let org_bucket: Arc<str> = Arc::from(org_bucket);
let bucket: Arc<str> = Arc::from(bucket);
let object: Arc<str> = Arc::from(object);
let version_id: Arc<str> = Arc::from(version_id);
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
let disk = disk.clone();
let task_opts = opts;
let org_bucket = org_bucket.clone();
let bucket = bucket.clone();
let object = object.clone();
let version_id = version_id.clone();
let slowtail_fault = slowtail_fault.clone();
AbortOnDropJoinHandle(tokio::spawn(async move {
let response_start = observe.then(Instant::now);
let result = if let Some(disk) = disk {
Self::record_read_version_call(&object, disk_index);
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
Self::record_metadata_slowtail_fault(&object, disk_index);
tokio::time::sleep(delay).await;
}
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
};
let elapsed = response_start.map(|start| start.elapsed());
(result, elapsed)
}))
});
let results = join_all(futures).await;
for join_result in results {
match join_result {
Ok((res, elapsed)) => match res {
Ok(file_info) => {
if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) {
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
}
ress.push(file_info);
errors.push(None);
}
Err(e) => {
if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) {
observations.push(MetadataFanoutObservation::from_error(&e, elapsed));
}
ress.push(FileInfo::default());
errors.push(Some(e));
}
},
Err(_join_err) => {
if let Some(observations) = &mut observations {
observations.push(MetadataFanoutObservation::from_error(&DiskError::Unexpected, Duration::ZERO));
}
ress.push(FileInfo::default());
errors.push(Some(DiskError::Unexpected));
}
}
}
let diagnostics = match (fanout_start, observations) {
(Some(fanout_start), Some(observations)) => MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations),
_ => MetadataFanoutDiagnostics::default(),
};
if observe {
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(
metadata_metrics_path(bucket.as_ref()),
scheduled_count,
scheduled_count,
0,
);
}
Ok((ress, errors, diagnostics))
}
#[allow(clippy::too_many_arguments)]
async fn read_all_fileinfo_early_stop(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
healing: bool,
incl_free_versions: bool,
allow_non_inline_data_read_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = Instant::now();
let mut ress = vec![FileInfo::default(); disks.len()];
let mut errors = vec![None; disks.len()];
let mut observations = Vec::with_capacity(disks.len());
let mut accumulator =
MetadataQuorumAccumulator::new(disks.len(), default_parity_count, true).with_requested_version_id(version_id);
let opts = ReadOptions {
incl_free_versions,
read_data,
healing,
};
let org_bucket: Arc<str> = Arc::from(org_bucket);
let bucket: Arc<str> = Arc::from(bucket);
let object: Arc<str> = Arc::from(object);
let version_id: Arc<str> = Arc::from(version_id);
let metrics_path = metadata_metrics_path(bucket.as_ref());
let mut join_set = JoinSet::new();
let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled();
let fanout_order = if bounded_fanout {
bounded_metadata_fanout_order(bucket.as_ref(), object.as_ref(), disks.len(), default_parity_count)
} else {
Vec::new()
};
let mut next_fanout_index = 0usize;
let mut scheduled_count = 0usize;
let mut force_full_wait = false;
let mut final_miss_reason_override = None;
let mut non_inline_candidate_eligible = None;
let mut single_pending_hedge_deadline = None;
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
let spawn_read_version =
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
let task_opts = opts;
let org_bucket = org_bucket.clone();
let bucket = bucket.clone();
let object = object.clone();
let version_id = version_id.clone();
let slowtail_fault = slowtail_fault.clone();
join_set.spawn(async move {
let response_start = Instant::now();
let result = if let Some(disk) = disk {
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&object);
Self::record_read_version_call(&object, index);
#[cfg(test)]
Self::read_version_fanout_barrier(&object, index).await;
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
Self::record_metadata_slowtail_fault(&object, index);
tokio::time::sleep(delay).await;
}
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
};
(index, result, response_start.elapsed())
});
};
if bounded_fanout {
let initial_target = accumulator.default_write_quorum().min(disks.len());
while next_fanout_index < initial_target {
let disk_index = fanout_order[next_fanout_index];
if let Some(disk) = disks.get(disk_index).cloned() {
spawn_read_version(&mut join_set, disk_index, disk);
scheduled_count = scheduled_count.saturating_add(1);
}
next_fanout_index = next_fanout_index.saturating_add(1);
}
} else {
for (index, disk) in disks.iter().cloned().enumerate() {
spawn_read_version(&mut join_set, index, disk);
scheduled_count = scheduled_count.saturating_add(1);
}
}
loop {
let mut defer_pending_inline_data_shard = false;
let result = if let Some(deadline) = single_pending_hedge_deadline.take() {
tokio::select! {
result = join_set.join_next() => result,
_ = async {
tokio::time::sleep_until(deadline).await;
#[cfg(test)]
rename_fanout_barrier::checkpoint(
object.as_ref(),
0,
rename_fanout_barrier::PHASE_NON_INLINE_HEDGE_TIMER,
)
.await;
} => {
if bounded_fanout
&& !force_full_wait
&& join_set.len() == 1
&& non_inline_candidate_eligible == Some(true)
&& !accumulator.candidate_has_read_reserve()
&& next_fanout_index < disks.len()
{
while next_fanout_index < disks.len() {
let disk_index = fanout_order[next_fanout_index];
next_fanout_index = next_fanout_index.saturating_add(1);
if let Some(disk) = disks.get(disk_index).cloned() {
spawn_read_version(&mut join_set, disk_index, disk);
scheduled_count = scheduled_count.saturating_add(1);
break;
}
}
}
continue;
}
}
} else {
join_set.join_next().await
};
let Some(result) = result else { break };
match result {
Ok((index, res, elapsed)) => match res {
Ok(file_info) => {
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
if allow_non_inline_data_read_early_stop {
accumulator.observe_file_info_at(index, &file_info);
} else {
accumulator.observe_file_info(&file_info);
}
if allow_non_inline_data_read_early_stop && non_inline_candidate_eligible.is_none() {
non_inline_candidate_eligible =
accumulator.candidate.as_ref().map(non_inline_data_read_candidate_is_safe);
}
if bounded_fanout
&& read_data
&& !force_full_wait
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
&& !(non_inline_candidate_eligible == Some(true)
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
{
force_full_wait = true;
final_miss_reason_override.get_or_insert(reason);
}
if let Some(slot) = ress.get_mut(index) {
*slot = file_info;
}
}
Err(err) => {
observations.push(MetadataFanoutObservation::from_error(&err, elapsed));
accumulator.observe_error(&err);
if let Some(slot) = errors.get_mut(index) {
*slot = Some(err);
}
}
},
Err(_) => {
let err = DiskError::Unexpected;
observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed()));
accumulator.observe_error(&err);
}
}
if !force_full_wait
&& let Some(decision) = accumulator
.early_stop_decision()
.or_else(|| accumulator.version_early_stop_decision())
{
let should_return_early = if read_data {
match accumulator.candidate.as_ref() {
Some(_candidate) if non_inline_candidate_eligible == Some(true) => {
accumulator.candidate_has_read_reserve()
}
Some(candidate) => match data_read_early_stop_inline_body_miss_reason(
bucket.as_ref(),
object.as_ref(),
candidate,
&ress,
disks,
)
.await
{
None => true,
Some(reason) => {
final_miss_reason_override = Some(reason);
if bounded_fanout
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
&& data_read_inline_missing_shards_are_pending(
candidate,
&ress,
&errors,
disks,
&fanout_order,
next_fanout_index,
)
{
defer_pending_inline_data_shard = true;
} else {
force_full_wait = true;
}
false
}
},
None => {
force_full_wait = true;
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
false
}
}
} else {
true
};
if should_return_early {
let saved_responses = if bounded_fanout {
disks.len().saturating_sub(observations.len())
} else {
join_set.len()
};
join_set.abort_all();
rustfs_io_metrics::record_get_object_metadata_early_stop_hit(metrics_path, decision.reason);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, saved_responses);
let mut cancelled_count = 0usize;
while let Some(join_result) = join_set.join_next().await {
match join_result {
Err(join_error) if join_error.is_cancelled() => {
cancelled_count = cancelled_count.saturating_add(1);
}
_ => {}
}
}
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(
metrics_path,
scheduled_count,
scheduled_count.saturating_sub(cancelled_count),
cancelled_count,
);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
return Ok((ress, errors, diagnostics));
}
}
let pending_responses = join_set.len();
let should_hedge_single_pending_inline_read = read_data
&& !force_full_wait
&& !defer_pending_inline_data_shard
&& pending_responses == 1
&& non_inline_candidate_eligible != Some(true)
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
let needs_non_inline_read_reserve = non_inline_candidate_eligible == Some(true)
&& !accumulator.candidate_has_read_reserve()
&& accumulator
.candidate_read_reserve_target()
.is_some_and(|reserve_target| scheduled_count < reserve_target || pending_responses == 0);
if bounded_fanout
&& !force_full_wait
&& (needs_non_inline_read_reserve || should_hedge_single_pending_inline_read)
&& next_fanout_index < disks.len()
{
let disk_index = fanout_order[next_fanout_index];
if let Some(disk) = disks.get(disk_index).cloned() {
spawn_read_version(&mut join_set, disk_index, disk);
scheduled_count = scheduled_count.saturating_add(1);
}
next_fanout_index = next_fanout_index.saturating_add(1);
} else if bounded_fanout && force_full_wait {
while next_fanout_index < disks.len() {
let disk_index = fanout_order[next_fanout_index];
if let Some(disk) = disks.get(disk_index).cloned() {
spawn_read_version(&mut join_set, disk_index, disk);
scheduled_count = scheduled_count.saturating_add(1);
}
next_fanout_index = next_fanout_index.saturating_add(1);
}
} else if bounded_fanout
&& !defer_pending_inline_data_shard
&& next_fanout_index < disks.len()
&& !accumulator.can_still_reach_early_stop_with_pending(pending_responses)
{
let disk_index = fanout_order[next_fanout_index];
if let Some(disk) = disks.get(disk_index).cloned() {
spawn_read_version(&mut join_set, disk_index, disk);
scheduled_count = scheduled_count.saturating_add(1);
}
next_fanout_index = next_fanout_index.saturating_add(1);
}
if bounded_fanout
&& !force_full_wait
&& !defer_pending_inline_data_shard
&& join_set.len() == 1
&& non_inline_candidate_eligible == Some(true)
&& !accumulator.candidate_has_read_reserve()
&& accumulator.can_still_reach_early_stop_with_pending(join_set.len())
&& next_fanout_index < disks.len()
{
single_pending_hedge_deadline = Some(tokio::time::Instant::now() + NON_INLINE_SINGLE_PENDING_HEDGE_DELAY);
}
}
let accumulator_miss_reason = accumulator.final_miss_reason();
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
_ => accumulator_miss_reason,
};
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
Ok((ress, errors, diagnostics))
}
pub(in crate::set_disk) async fn read_all_xl(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
read_data: bool,
incl_free_vers: bool,
) -> (Vec<FileInfo>, Vec<Option<DiskError>>) {
let (fileinfos, errs) = Self::read_all_raw_file_info(disks, bucket, object, read_data).await;
Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await
}
pub(crate) async fn load_file_info_versions_exact(
&self,
bucket: &str,
object: &str,
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
self.load_file_info_versions_for_cleanup(bucket, object, false).await
}
pub(crate) async fn load_file_info_versions_for_tier_cleanup(
&self,
bucket: &str,
object: &str,
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
self.load_file_info_versions_for_cleanup(bucket, object, true).await
}
async fn load_file_info_versions_for_cleanup(
&self,
bucket: &str,
object: &str,
retain_unconfirmed_tier_references: bool,
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
let disk_object = rustfs_utils::path::encode_dir_object(object);
let disks = self.get_disks_internal().await;
if disks.is_empty() {
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
}
let read_quorum = disks.len().div_ceil(2).max(1);
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
if let Some(err) = errs
.iter()
.flatten()
.find(|err| !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound))
{
return Err(to_object_err(err.clone().into(), vec![bucket, object]));
}
if raw_fileinfos.iter().all(Option::is_none) {
return Ok(None);
}
let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len());
type TransitionCopy = (FileInfo, Option<crate::services::tier::tier::TierDestinationId>);
let mut transition_copies: std::collections::HashMap<Option<Uuid>, Vec<TransitionCopy>> =
std::collections::HashMap::new();
let decode_error = |err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}"));
for raw_fileinfo in raw_fileinfos.into_iter().flatten() {
let meta = FileMeta::load(&raw_fileinfo.buf)
.map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?;
let versions = meta.get_all_file_info_versions(bucket, object, true).map_err(decode_error)?;
for version in versions.versions.into_iter().chain(versions.free_versions) {
if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE {
continue;
}
let destination =
crate::services::tier::tier::tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?;
transition_copies
.entry(version.version_id.filter(|id| !id.is_nil()))
.or_default()
.push((version, destination));
}
shallow_versions.push(meta.versions);
}
for copies in transition_copies
.values()
.filter(|copies| copies.iter().any(|(_, destination)| destination.is_some()))
{
let (first, destination) = &copies[0];
if copies.iter().any(|(copy, identity)| {
identity != destination
|| copy.transition_version_state != first.transition_version_state
|| copy.transition_version != first.transition_version
|| copy.transition_tier != first.transition_tier
|| copy.transitioned_objname != first.transitioned_objname
}) {
return Err(Error::other("exact transition metadata has not converged across physical copies"));
}
}
if shallow_versions.len() < read_quorum {
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
}
let versions = merge_file_meta_versions(read_quorum, true, 0, &shallow_versions);
if versions.is_empty() {
return Err(Error::other(format!(
"exact object metadata read returned no quorum versions for {bucket}/{object}"
)));
}
let mut file_info_versions = FileMeta {
versions,
..Default::default()
}
.get_all_file_info_versions(bucket, object, true)
.map_err(decode_error)?;
if retain_unconfirmed_tier_references {
file_info_versions.versions.extend(
transition_copies
.into_values()
.flatten()
.map(|(version, _)| version)
.filter(|version| !version.tier_free_version()),
);
}
for file_info in file_info_versions
.versions
.iter()
.chain(file_info_versions.free_versions.iter())
{
file_info
.validate_for_metadata_read()
.map_err(|err| Error::other(format!("exact object versions validation failed for {bucket}/{object}: {err}")))?;
}
Ok(Some(file_info_versions))
}
pub(in crate::set_disk) async fn read_all_raw_file_info(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
read_data: bool,
) -> (Vec<Option<RawFileInfo>>, Vec<Option<DiskError>>) {
let mut ress = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
let mut futures = Vec::with_capacity(disks.len());
for disk in disks.iter() {
futures.push(async move {
if let Some(disk) = disk {
disk.read_xl(bucket, object, read_data).await
} else {
Err(DiskError::DiskNotFound)
}
});
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(res) => {
ress.push(Some(res));
errors.push(None);
}
Err(e) => {
ress.push(None);
errors.push(Some(e));
}
}
}
(ress, errors)
}
pub(in crate::set_disk) async fn pick_latest_quorum_files_info(
fileinfos: Vec<Option<RawFileInfo>>,
errs: Vec<Option<DiskError>>,
bucket: &str,
object: &str,
read_data: bool,
incl_free_vers: bool,
) -> (Vec<FileInfo>, Vec<Option<DiskError>>) {
let mut metadata_array = vec![None; fileinfos.len()];
let mut meta_file_infos = vec![FileInfo::default(); fileinfos.len()];
let mut metadata_shallow_versions = vec![None; fileinfos.len()];
let mut v2_bufs = {
if !read_data {
vec![Vec::new(); fileinfos.len()]
} else {
Vec::new()
}
};
let mut errs = errs;
for (idx, info_op) in fileinfos.iter().enumerate() {
if let Some(info) = info_op {
if !read_data {
v2_bufs[idx] = info.buf.clone();
}
let xlmeta = match FileMeta::load(&info.buf) {
Ok(res) => res,
Err(err) => {
errs[idx] = Some(err.into());
continue;
}
};
metadata_array[idx] = Some(xlmeta);
meta_file_infos[idx] = FileInfo::default();
}
}
for (idx, info_op) in metadata_array.iter().enumerate() {
if let Some(info) = info_op {
metadata_shallow_versions[idx] = Some(info.versions.clone());
}
}
let shallow_versions: Vec<Vec<FileMetaShallowVersion>> = metadata_shallow_versions.iter().flatten().cloned().collect();
let read_quorum = fileinfos.len().div_ceil(2);
let versions = merge_file_meta_versions(read_quorum, false, 1, &shallow_versions);
let meta = FileMeta {
versions,
..Default::default()
};
let vid = match meta.into_fileinfo_without_part_checksums(bucket, object, "", true, incl_free_vers) {
Ok(finfo) if file_info_is_valid_for_metadata(&finfo) => finfo.version_id.unwrap_or(Uuid::nil()),
_ => match meta
.versions
.iter()
.find(|v| !v.header.free_version())
.and_then(|v| v.header.version_id)
{
Some(id) => id,
None => {
for item in errs.iter_mut() {
if item.is_none() {
*item = Some(DiskError::FileCorrupt);
}
}
return (meta_file_infos, errs);
}
},
};
for (idx, meta_op) in metadata_array.iter().enumerate() {
if let Some(meta) = meta_op {
match meta.into_fileinfo_without_part_checksums(
bucket,
object,
vid.to_string().as_str(),
read_data,
incl_free_vers,
) {
Ok(res) => match res.validate_for_metadata_read() {
Ok(_) => meta_file_infos[idx] = res,
Err(err) => errs[idx] = Some(err.into()),
},
Err(err) => errs[idx] = Some(err.into()),
}
}
}
(meta_file_infos, errs)
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn read_multiple_files(
disks: &[Option<DiskStore>],
req: ReadMultipleReq,
read_quorum: usize,
) -> Vec<ReadMultipleResp> {
let mut futures = Vec::with_capacity(disks.len());
let empty_quorum_result = || {
req.files
.iter()
.map(|want| ReadMultipleResp {
bucket: req.bucket.clone(),
prefix: req.prefix.clone(),
file: want.clone(),
exists: false,
error: Error::ErasureReadQuorum.to_string(),
data: Vec::new(),
mod_time: None,
})
.collect::<Vec<_>>()
};
for disk in disks.iter() {
let disk = disk.clone();
let req = req.clone();
futures.push(async move {
if let Some(disk) = disk {
disk.read_multiple(req).await
} else {
Err(DiskError::DiskNotFound)
}
});
}
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await {
Ok(collected) => collected,
Err(()) => return empty_quorum_result(),
};
let mut ret = Vec::with_capacity(req.files.len());
for want in req.files.iter() {
let mut quorum = 0;
let mut get_res = ReadMultipleResp::default();
for res in ress.iter() {
if res.is_none() {
continue;
}
let disk_res = res.as_ref().unwrap();
for resp in disk_res.iter() {
if !resp.error.is_empty() || !resp.exists {
continue;
}
if &resp.file != want || resp.bucket != req.bucket || resp.prefix != req.prefix {
continue;
}
quorum += 1;
if get_res.mod_time > resp.mod_time || get_res.data.len() > resp.data.len() {
continue;
}
get_res = resp.clone();
}
}
if quorum < read_quorum {
get_res.exists = false;
get_res.error = Error::ErasureReadQuorum.to_string();
get_res.data = Vec::new();
}
ret.push(get_res);
}
ret
}
}
const ENV_HEAL_DANGLING_DELETE_GRACE_SECS: &str = "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS";
const DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS: u64 = 3600;
fn dangling_delete_grace() -> time::Duration {
let secs = rustfs_utils::get_env_u64(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS);
time::Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX))
}
enum OrphanDirScan {
Scanned {
blocked: HashSet<String>,
dirs: Vec<String>,
committed_files: Vec<String>,
},
Unreadable,
Missing,
}
const ORPHAN_PURGE_BACKOFF: Duration = Duration::from_secs(60);
const ORPHAN_PURGE_BACKOFF_MAX_ENTRIES: usize = 4096;
fn block_orphan_dir_chain(blocked: &mut HashSet<String>, root: &str, dir: &str) {
let mut current = dir;
loop {
if !blocked.insert(current.to_owned()) || current == root {
return;
}
let Some((parent, _)) = current.rsplit_once(SLASH_SEPARATOR) else {
return;
};
current = parent;
}
}
fn is_safe_orphan_dir_entry(entry: &str) -> bool {
let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
!component.is_empty()
&& component != "."
&& component != ".."
&& !component.contains(SLASH_SEPARATOR)
&& !component.contains('\\')
}
fn is_committed_delete_marker(entry: &str) -> bool {
entry
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil()))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(in crate::set_disk) enum RenameConvergence {
AllSuccessIdentical,
PartialCommit,
SignatureDivergent,
Unknown,
}
impl RenameConvergence {
pub(in crate::set_disk) fn needs_heal(&self) -> bool {
matches!(self, Self::PartialCommit | Self::SignatureDivergent)
}
}
pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) online_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) capacity_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) convergence: RenameConvergence,
pub(in crate::set_disk) data_dir: Option<Uuid>,
pub(in crate::set_disk) cleanup_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) old_current_size: Option<OldCurrentSize>,
pub(in crate::set_disk) committed_file_info: FileInfo,
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
}
pub(in crate::set_disk) struct RenameTailCleanup {
pub(in crate::set_disk) disk_index: usize,
pub(in crate::set_disk) disk: DiskStore,
pub(in crate::set_disk) old_data_dir: Uuid,
}
pub(in crate::set_disk) struct RenameTailOutcome {
pub(in crate::set_disk) convergence: RenameConvergence,
pub(in crate::set_disk) cleanup: Vec<RenameTailCleanup>,
}
const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback";
#[derive(Clone, Copy)]
enum RenameDispatchState {
NotDispatched,
RejectedBeforePublication,
MayHavePublished,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum RenameRollbackOutcome {
NotAttempted(DiskError),
RejectedBeforePublication(DiskError),
Indeterminate(DiskError),
Succeeded,
Failed(DiskError),
Panicked,
Cancelled,
}
impl RenameRollbackOutcome {
fn stage(&self) -> &'static str {
match self {
Self::NotAttempted(_) => "rename_not_dispatched",
Self::RejectedBeforePublication(_) => "rename_rejected_before_publication",
Self::Indeterminate(_) => "rename_indeterminate",
Self::Succeeded => "undo_succeeded",
Self::Failed(_) => "undo_failed",
Self::Panicked => "undo_panicked",
Self::Cancelled => "undo_cancelled",
}
}
fn undo_attempted(&self) -> bool {
matches!(self, Self::Succeeded | Self::Failed(_) | Self::Panicked | Self::Cancelled)
}
fn needs_recovery(&self) -> bool {
matches!(self, Self::Indeterminate(_) | Self::Failed(_) | Self::Panicked | Self::Cancelled)
}
}
#[derive(Debug, Clone)]
struct RenameRollbackDiskOutcome {
disk_index: usize,
rollback_dir: Option<Uuid>,
outcome: RenameRollbackOutcome,
}
#[derive(Debug)]
struct RenameRollbackReport {
disks: Vec<RenameRollbackDiskOutcome>,
}
#[derive(Clone, Default)]
pub(in crate::set_disk) struct RenameRollbackReceipt(Arc<OnceLock<RenameRollbackReport>>);
impl RenameRollbackReceipt {
pub(in crate::set_disk) fn is_incomplete(&self) -> bool {
self.0
.get()
.is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.needs_recovery()))
}
}
struct RenameRollbackOwnership {
receipt: Option<RenameRollbackReceipt>,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
}
async fn inspect_incomplete_rename_rollback(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
submitter: ReadRepairAdmissionSubmitter,
) -> ReadRepairAdmissionOutcome {
let location = disks.iter().flatten().next().map(|disk| disk.get_disk_location());
let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(HealChannelPriority::High),
location.as_ref().and_then(|location| location.pool_idx),
location.as_ref().and_then(|location| location.set_idx),
);
request.dry_run = Some(true);
request.remove_corrupted = Some(false);
request.recreate_missing = Some(false);
request.update_parity = Some(false);
request.recursive = Some(false);
match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await {
Ok(result) => result,
Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()),
}
}
fn rename_rollback_task_outcome(
result: std::result::Result<disk::error::Result<()>, tokio::task::JoinError>,
) -> RenameRollbackOutcome {
match result {
Ok(Ok(())) => RenameRollbackOutcome::Succeeded,
Ok(Err(err)) => RenameRollbackOutcome::Failed(err),
Err(err) if err.is_panic() => RenameRollbackOutcome::Panicked,
Err(_) => RenameRollbackOutcome::Cancelled,
}
}
async fn rollback_failed_rename(
disks: &[Option<DiskStore>],
file_infos: Vec<FileInfo>,
errs: &[Option<DiskError>],
dispatch_states: &[RenameDispatchState],
rollback_dirs: &[Option<Uuid>],
dst: (&str, &str),
ownership: RenameRollbackOwnership,
) {
let RenameRollbackOwnership {
receipt,
namespace_commit_guard,
} = ownership;
let owned_disks = disks.to_vec();
let owned_errs = errs.to_vec();
let owned_dispatch_states = dispatch_states.to_vec();
let owned_dirs = rollback_dirs.to_vec();
let owned_dst = (dst.0.to_string(), dst.1.to_string());
let coordinator_failure_receipt = receipt.clone();
let rollback = tokio::spawn(async move {
let disks = owned_disks.as_slice();
let errs = owned_errs.as_slice();
let dispatch_states = owned_dispatch_states.as_slice();
let rollback_dirs = owned_dirs.as_slice();
let dst = (owned_dst.0.as_str(), owned_dst.1.as_str());
let mut file_infos = file_infos;
let (bucket, object) = dst;
let mut outcomes = Vec::with_capacity(disks.len());
let mut tasks = Vec::with_capacity(disks.len());
for (disk_index, disk) in disks.iter().enumerate() {
let rollback_dir = rollback_dirs[disk_index];
let outcome = match &errs[disk_index] {
Some(err) => match dispatch_states[disk_index] {
RenameDispatchState::NotDispatched => RenameRollbackOutcome::NotAttempted(err.clone()),
RenameDispatchState::RejectedBeforePublication => {
RenameRollbackOutcome::RejectedBeforePublication(err.clone())
}
RenameDispatchState::MayHavePublished => RenameRollbackOutcome::Indeterminate(err.clone()),
},
None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound),
};
outcomes.push(RenameRollbackDiskOutcome {
disk_index,
rollback_dir,
outcome,
});
if errs[disk_index].is_some() {
continue;
}
let Some(disk) = disk.clone() else {
continue;
};
let fi = std::mem::take(&mut file_infos[disk_index]);
let bucket = bucket.to_string();
let object = object.to_string();
let disk_namespace_commit_guard = namespace_commit_guard.clone();
let task = tokio::spawn(async move {
let _namespace_commit_guard = disk_namespace_commit_guard.clone();
#[allow(clippy::let_unit_value)]
let _task_guard = SetDisks::rename_fanout_task_guard(&object);
SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
#[cfg(test)]
rollback_fault_injection::before_undo(&object, disk_index)?;
disk.undo_write_with_namespace_owner(
&bucket,
&object,
fi,
DeleteOptions {
undo_write: true,
old_data_dir: rollback_dir,
..Default::default()
},
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
)
.await
});
tasks.push(async move { (disk_index, task.await) });
}
#[cfg(test)]
rollback_fault_injection::after_undo_dispatch(object);
let _namespace_commit_guard = namespace_commit_guard;
for (disk_index, result) in join_all(tasks).await {
outcomes[disk_index].outcome = rename_rollback_task_outcome(result);
}
record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await;
});
if rollback.await.is_err() {
record_indeterminate_rename(disks, dst, coordinator_failure_receipt).await;
}
}
async fn record_indeterminate_rename(disks: &[Option<DiskStore>], dst: (&str, &str), receipt: Option<RenameRollbackReceipt>) {
let outcomes = disks
.iter()
.enumerate()
.map(|(disk_index, disk)| RenameRollbackDiskOutcome {
disk_index,
rollback_dir: None,
outcome: if disk.is_some() {
RenameRollbackOutcome::Indeterminate(DiskError::Unexpected)
} else {
RenameRollbackOutcome::NotAttempted(DiskError::DiskNotFound)
},
})
.collect();
record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await;
}
async fn record_rename_rollback_outcomes(
disks: &[Option<DiskStore>],
outcomes: Vec<RenameRollbackDiskOutcome>,
dst: (&str, &str),
receipt: Option<RenameRollbackReceipt>,
) {
let (bucket, object) = dst;
let attempted = outcomes.iter().filter(|disk| disk.outcome.undo_attempted()).count();
let failed = outcomes
.iter()
.filter(|disk| disk.outcome.undo_attempted() && disk.outcome.needs_recovery())
.count();
let indeterminate = outcomes
.iter()
.filter(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(_)))
.count();
let succeeded = attempted - failed;
for disk in &outcomes {
counter!("rustfs_rename_rollback_disks_total", "stage" => disk.outcome.stage()).increment(1);
if disk.outcome.needs_recovery() {
let location = disks[disk.disk_index].as_ref().map(|disk| disk.get_disk_location());
warn!(
event = EVENT_SET_DISK_RENAME_ROLLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "recovery_required",
stage = disk.outcome.stage(),
pool_index = ?location.as_ref().and_then(|location| location.pool_idx),
set_index = ?location.as_ref().and_then(|location| location.set_idx),
disk_index = disk.disk_index,
bucket,
object,
rollback_dir = ?disk.rollback_dir,
outcome = ?disk.outcome,
attempted,
succeeded,
failed,
indeterminate,
"rename rollback incomplete; preserve recovery material"
);
}
}
if let Some(receipt) = receipt {
let _ = receipt.0.set(RenameRollbackReport { disks: outcomes });
}
if failed > 0 || indeterminate > 0 {
let result = inspect_incomplete_rename_rollback(disks, bucket, object, send_read_repair_heal_request).await;
let admission = match &result {
ReadRepairAdmissionOutcome::Response(response) => response.result_label(),
ReadRepairAdmissionOutcome::Failed(_) => "failed",
};
counter!("rustfs_rename_rollback_inspection_total", "admission" => admission).increment(1);
warn!(
event = EVENT_SET_DISK_RENAME_ROLLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "recovery_required",
stage = "inspection_admission",
bucket,
object,
attempted,
succeeded,
failed,
indeterminate,
admission,
outcome = ?result,
"rename rollback inspection requested; recovery remains incomplete"
);
}
}
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
rollback_receipt: Option<RenameRollbackReceipt>,
}
impl<'a> RenameDataFenceOptions<'a> {
pub(in crate::set_disk) fn new(
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
) -> Self {
Self {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: None,
namespace_commit_guard: None,
rollback_receipt: None,
}
}
pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
self.rollback_receipt = Some(receipt);
self
}
pub(in crate::set_disk) fn with_publication_scope(
mut self,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
) -> Self {
self.scanner_publication_commit_scope = scanner_publication_commit_scope;
self
}
pub(in crate::set_disk) fn with_namespace_commit_guard(
mut self,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
) -> Self {
self.namespace_commit_guard = namespace_commit_guard;
self
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>,
RenameConvergence,
Option<Uuid>,
Vec<Option<DiskStore>>,
Option<OldCurrentSize>,
);
fn put_rename_early_ack_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
}
impl RenameDataCommit {
pub(in crate::set_disk) fn needs_immediate_heal(&self) -> bool {
self.tail_drain.is_none() && self.convergence.needs_heal()
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
(
self.online_disks,
self.convergence,
self.data_dir,
self.cleanup_disks,
self.old_current_size,
)
}
}
pub(in crate::set_disk) async fn finish_rename_tail_heal<
Guards,
Finalize,
FinalizeFuture,
Cleanup,
CleanupFuture,
Submit,
SubmitFuture,
>(
tail_drain: tokio::task::JoinHandle<Option<RenameTailOutcome>>,
guard_release: tokio::sync::oneshot::Receiver<bool>,
guards: Guards,
request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
finalize: Finalize,
cleanup: Cleanup,
submit: Submit,
) where
Guards: Send,
Finalize: FnOnce() -> FinalizeFuture + Send,
FinalizeFuture: Future<Output = ()> + Send,
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>) -> CleanupFuture + Send,
CleanupFuture: Future<Output = ()> + Send,
Submit: FnOnce(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> SubmitFuture + Send,
SubmitFuture: Future<Output = ()> + Send,
{
let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await {
Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true),
Ok(None) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "missing_convergence",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
"committed rename tail drain completed without convergence"
);
(true, Vec::new(), false)
}
Err(err) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "failed",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
error = %err,
"rename tail drain failed"
);
(true, Vec::new(), false)
}
};
if matches!(guard_release.await, Ok(false)) {
drop(guards);
return;
}
finalize().await;
if tail_complete {
cleanup(guards, tail_cleanup).await;
} else {
drop(guards);
}
if needs_heal {
submit(request).await;
}
}
async fn run_scanner_publication_delete_owner<F, Fut>(
scope: Option<crate::object_api::ScannerPublicationCommitScope>,
operation: F,
) -> disk::error::Result<()>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: Future<Output = disk::error::Result<()>> + Send + 'static,
{
if scope.is_none() {
return operation().await;
}
if let Some(scope) = scope.as_ref() {
scope.attach_mutation_owner();
}
tokio::spawn(async move {
let result = operation().await;
if let Some(scope) = scope.as_ref() {
if result.is_ok() {
let _ = scope.mark_committed();
} else {
let _ = scope.mark_indeterminate();
}
}
result
})
.await
.map_err(|_| DiskError::other("scanner publication delete owner failed"))?
}
impl SetDisks {
pub(in crate::set_disk) fn default_read_quorum(&self) -> usize {
self.set_drive_count - self.default_parity_count
}
pub(crate) fn default_write_quorum(&self) -> usize {
let mut data_count = self.set_drive_count - self.default_parity_count;
if data_count == self.default_parity_count {
data_count += 1
}
data_count
}
pub(in crate::set_disk) async fn prepare_quota_mutation_fences(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
let fence_path = fence_path.clone();
async move {
let disk = disk?;
match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await {
Ok(token) => Some((disk, token)),
Err(_) => None,
}
}
}))
.await;
if results.iter().flatten().count() < write_quorum {
for (disk, token) in results.iter().flatten() {
let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await;
}
return Err(StorageError::ErasureWriteQuorum);
}
let mut fenced_disks = Vec::with_capacity(results.len());
let mut tokens = Vec::with_capacity(results.len());
for result in results {
match result {
Some((disk, token)) => {
fenced_disks.push(Some(disk));
tokens.push(Some(token));
}
None => {
fenced_disks.push(None);
tokens.push(None);
}
}
}
Ok((fenced_disks, tokens))
}
pub(in crate::set_disk) async fn release_quota_mutation_fences(
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<()> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
let disk = disk.as_ref()?.clone();
let token = (*token)?;
let fence_path = fence_path.clone();
Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await })
}))
.await;
if results.iter().filter(|result| result.is_ok()).count() < write_quorum {
return Err(StorageError::ErasureWriteQuorum);
}
Ok(())
}
pub(in crate::set_disk) fn assign_rename_data_indexes(file_infos: &mut [FileInfo]) {
for (index, file_info) in file_infos.iter_mut().enumerate() {
if file_info.erasure.index == 0 {
file_info.erasure.index = index + 1;
}
}
}
fn rename_data_commit_from_observations(
disks: &[Option<DiskStore>],
file_infos: &[FileInfo],
disk_versions: &[Option<Vec<u8>>],
errs: &[Option<DiskError>],
cleanup_data_dirs: &[Option<Uuid>],
old_current_sizes: &[Option<OldCurrentSize>],
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum);
let convergence = Self::classify_rename_convergence(disk_versions, errs);
let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum);
let online_disks = Self::eval_disks(disks, errs);
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
let committed_file_info = file_infos.get(committed_slot).cloned().ok_or(DiskError::Unexpected)?;
let cleanup_disks = if let Some(data_dir) = data_dir {
disks
.iter()
.zip(errs.iter())
.zip(cleanup_data_dirs.iter())
.map(|((disk, err), old_data_dir)| {
if err.is_none() && *old_data_dir == Some(data_dir) {
disk.clone()
} else {
None
}
})
.collect()
} else {
vec![None; disks.len()]
};
Ok(RenameDataCommit {
capacity_disks: disks.to_vec(),
online_disks,
convergence,
data_dir,
cleanup_disks,
old_current_size,
committed_file_info,
tail_drain: None,
})
}
pub(in crate::set_disk) async fn abort_quota_reservation_after_fence(
reservation: crate::bucket::quota::reservation::QuotaReservation,
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
fenced: bool,
) {
let safe_to_abort = !fenced
|| Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum)
.await
.is_ok();
if safe_to_abort {
reservation.abort().await;
} else {
reservation.defer_after_fence();
}
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
#[allow(clippy::type_complexity)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn rename_data(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: &[FileInfo],
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<RenameDataLegacyTuple> {
Self::rename_data_owned(
disks,
(src_bucket, src_object),
file_infos.to_vec(),
(dst_bucket, dst_object),
write_quorum,
false,
)
.await
.map(RenameDataCommit::into_legacy_tuple)
}
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
disk: Option<&DiskStore>,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Option<Uuid>> {
let Some(disk) = disk else {
return Ok(None);
};
if disk.is_local() {
return Ok(None);
}
let Some(tokens) = scanner_publication_lease_tokens else {
return Ok(None);
};
let host = disk.endpoint().grid_host();
tokens
.get(&host)
.copied()
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
.map(Some)
}
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
disks: &[Option<DiskStore>],
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Vec<Option<Uuid>>> {
disks
.iter()
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
.collect()
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
async fn rename_data_owned_early_ack_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: _scanner_publication_commit_scope,
namespace_commit_guard,
rollback_receipt,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
.find_map(|(disk, file_info)| disk.as_ref().map(|_| file_info))
{
if file_info.is_canonical_delete_marker() {
file_info.validate_for_metadata_read()?;
} else {
file_info.validate_for_erasure_write()?;
}
}
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let (commit_tx, commit_rx) = tokio::sync::oneshot::channel();
let coordinator_failure_receipt = rollback_receipt.clone();
let tail_drain = tokio::spawn({
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
let fanout_dst_object = dst_object.clone();
async move {
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let mut tasks = JoinSet::new();
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
.into_iter()
.zip(file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
{
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
let dst_object = fanout_dst_object.clone();
let file_info = file_info.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard.clone();
let mut dispatch_state = RenameDispatchState::NotDispatched;
let result = std::panic::AssertUnwindSafe(async {
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let is_delete_marker = file_info.is_canonical_delete_marker();
let mut file_info = file_info.clone();
if file_info.erasure.index == 0 {
file_info.erasure.index = i + 1;
}
if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) {
return Err(DiskError::FileCorrupt);
}
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
if let Some(err) = Self::rename_injected_error(&dst_object, i) {
return Err(err);
}
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
&src_object,
&file_info,
&dst_bucket,
&dst_object,
crate::disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
..Default::default()
},
)
.await;
let rejected_before_publication = observed.rejected_before_publication();
let result = observed.result;
#[cfg(test)]
if result.is_ok() {
rollback_fault_injection::after_rename(&dst_object, i)?;
}
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
duration_ms,
);
let position = if result.is_ok() {
let rank = successful_rename_completion_rank
.as_ref()
.map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1)
.unwrap_or(1);
if rank <= write_quorum {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST
} else {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL
}
} else {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR
};
rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms);
}
if rejected_before_publication {
dispatch_state = RenameDispatchState::RejectedBeforePublication;
}
result
})
.catch_unwind()
.await;
(i, dispatch_state, result)
});
}
let mut commit_tx = Some(commit_tx);
let mut sent_commit = false;
let mut success_count = 0usize;
let mut fanout_panic = 0usize;
let mut results_seen = 0usize;
let mut errs = vec![Some(DiskError::DiskNotFound); disk_count];
let mut dispatch_states = vec![RenameDispatchState::MayHavePublished; disk_count];
let mut disk_versions = vec![None; disk_count];
let mut data_dirs = vec![None; disk_count];
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let mut capacity_scope_generation = None;
let mut cleanup_at_snapshot = vec![false; disk_count];
while let Some(joined) = tasks.join_next().await {
results_seen += 1;
match joined {
Ok((idx, dispatch_state, Ok(Ok(res)))) => {
dispatch_states[idx] = dispatch_state;
data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir);
cleanup_data_dirs[idx] = res.cleanup_data_dir;
disk_versions[idx] = res.sign;
old_current_sizes[idx] = res.old_current_size;
errs[idx] = None;
success_count += 1;
}
Ok((idx, dispatch_state, Ok(Err(err)))) => {
dispatch_states[idx] = dispatch_state;
errs[idx] = Some(err);
}
Ok((idx, dispatch_state, Err(_))) => {
dispatch_states[idx] = dispatch_state;
errs[idx] = Some(DiskError::Unexpected);
fanout_panic += 1;
}
Err(_) => {
fanout_panic += 1;
}
}
if !sent_commit && success_count >= write_quorum {
let snapshot_commit = Self::rename_data_commit_from_observations(
&coordinator_disks,
&file_infos,
&disk_versions,
&errs,
&cleanup_data_dirs,
&old_current_sizes,
write_quorum,
);
if let Ok(commit) = snapshot_commit.as_ref() {
capacity_scope_generation = Some(current_dirty_generation());
cleanup_at_snapshot = commit.cleanup_disks.iter().map(Option::is_some).collect();
}
if let Some(commit_tx) = commit_tx.take() {
let _ = commit_tx.send(snapshot_commit);
}
sent_commit = true;
}
}
#[cfg(test)]
rollback_fault_injection::after_fanout(&fanout_dst_object);
if rustfs_io_metrics::put_stage_metrics_enabled() {
let fanout_success = errs.iter().filter(|err| err.is_none()).count();
let fanout_error = errs.len().saturating_sub(fanout_success + fanout_panic);
rustfs_io_metrics::record_put_rename_quorum_wait_fanout(
results_seen,
write_quorum,
fanout_success,
fanout_error,
fanout_panic,
);
}
if !sent_commit {
let ret_err =
reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum).unwrap_or(DiskError::Unexpected);
rollback_failed_rename(
&coordinator_disks,
file_infos,
&errs,
&dispatch_states,
&data_dirs,
(&fanout_dst_bucket, &fanout_dst_object),
RenameRollbackOwnership {
receipt: rollback_receipt,
namespace_commit_guard,
},
)
.await;
if let Some(commit_tx) = commit_tx.take() {
let _ = commit_tx.send(Err(ret_err));
}
return None;
}
let final_convergence = Self::classify_rename_convergence(&disk_versions, &errs);
let final_data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
let tail_cleanup = coordinator_disks
.iter()
.zip(errs.iter())
.zip(cleanup_data_dirs.iter())
.zip(cleanup_at_snapshot.iter())
.enumerate()
.filter_map(|(disk_index, (((disk, err), old_data_dir), cleanup_at_snapshot))| {
if *cleanup_at_snapshot || err.is_some() || *old_data_dir != final_data_dir {
return None;
}
Some(RenameTailCleanup {
disk_index,
disk: disk.clone()?,
old_data_dir: (*old_data_dir)?,
})
})
.collect();
let mut backup_reclaims = Vec::new();
for (idx, disk) in coordinator_disks.iter().enumerate() {
if errs[idx].is_some() {
continue;
}
let Some(rollback_dir) = data_dirs[idx] else {
continue;
};
if cleanup_data_dirs[idx] == Some(rollback_dir) {
continue;
}
let Some(disk) = disk.clone() else {
continue;
};
let dst_bucket = fanout_dst_bucket.clone();
let dst_object = fanout_dst_object.clone();
backup_reclaims.push(tokio::spawn(async move {
let backup_path = format!("{dst_object}/{rollback_dir}/{STORAGE_FORMAT_FILE_BACKUP}");
disk.delete(&dst_bucket, &backup_path, DeleteOptions::default()).await
}));
}
for result in join_all(backup_reclaims).await {
match result {
Ok(Ok(())) => {}
Ok(Err(DiskError::FileNotFound | DiskError::VolumeNotFound)) => {}
Ok(Err(err)) => {
warn!(
dst_bucket = %fanout_dst_bucket,
dst_object = %fanout_dst_object,
error = %err,
"rollback backup reclamation failed after committed rename"
);
}
Err(join_err) => {
warn!(
dst_bucket = %fanout_dst_bucket,
dst_object = %fanout_dst_object,
error = %join_err,
"rollback backup reclamation task failed after committed rename"
);
}
}
}
if capacity_scope_generation.is_some_and(|generation| current_dirty_generation() != generation) {
let scope = capacity_scope_from_disks(&coordinator_disks);
if !scope.disks.is_empty() {
let _ = record_global_dirty_scope(scope);
}
}
Some(RenameTailOutcome {
convergence: final_convergence,
cleanup: tail_cleanup,
})
}
});
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let commit = match commit_rx.await {
Ok(commit) => commit,
Err(_) => {
record_indeterminate_rename(disks, (&dst_bucket, &dst_object), coordinator_failure_receipt).await;
return Err(DiskError::Unexpected);
}
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
quorum_wait_started,
);
commit.map(|mut commit| {
commit.tail_drain = Some(tail_drain);
commit
})
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
pub(in crate::set_disk) async fn rename_data_owned(
disks: &[Option<DiskStore>],
src: (&str, &str),
file_infos: Vec<FileInfo>,
dst: (&str, &str),
write_quorum: usize,
allow_early_ack: bool,
) -> disk::error::Result<RenameDataCommit> {
Self::rename_data_owned_with_fence(
disks,
src,
file_infos,
dst,
allow_early_ack,
RenameDataFenceOptions::new(write_quorum, None),
)
.await
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
disks: &[Option<DiskStore>],
src: (&str, &str),
file_infos: Vec<FileInfo>,
dst: (&str, &str),
allow_early_ack: bool,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
let (src_bucket, src_object) = src;
let (dst_bucket, dst_object) = dst;
if allow_early_ack && put_rename_early_ack_enabled() {
return Self::rename_data_owned_early_ack_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
fence_options,
)
.await;
}
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope,
namespace_commit_guard,
rollback_receipt,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
.find_map(|(disk, file_info)| disk.as_ref().map(|_| file_info))
{
if file_info.is_canonical_delete_marker() {
file_info.validate_for_metadata_read()?;
} else {
file_info.validate_for_erasure_write()?;
}
}
let mut errs = Vec::with_capacity(disks.len());
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
let fanout_dst_object = dst_object.clone();
let fanout_publication_scope = scanner_publication_commit_scope.clone();
let fanout_namespace_commit_guard = namespace_commit_guard.clone();
let fanout = tokio::spawn(async move {
let _fanout_publication_scope = fanout_publication_scope;
let _namespace_commit_guard = fanout_namespace_commit_guard.clone();
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let futures = fanout_disks
.into_iter()
.zip(fanout_file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let publication_scope = scanner_publication_commit_scope.clone();
let namespace_commit_guard = fanout_namespace_commit_guard.clone();
async move {
let mut dispatch_state = RenameDispatchState::NotDispatched;
let result = std::panic::AssertUnwindSafe(async {
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let is_delete_marker = file_info.is_canonical_delete_marker();
let mut local_file_info;
let file_info = if file_info.erasure.index == 0 {
local_file_info = file_info.clone();
local_file_info.erasure.index = i + 1;
&local_file_info
} else {
file_info
};
if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) {
return Err(DiskError::FileCorrupt);
}
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
if let Some(err) = Self::rename_injected_error(&dst_object, i) {
return Err(err);
}
if let Some(scope) = publication_scope.as_ref()
&& !scope.can_commit()
{
let _ = scope.mark_indeterminate();
return Err(DiskError::other(
"scanner publication commit scope deadline or cancellation reached",
));
}
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
crate::disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
..Default::default()
},
)
.await;
let rejected_before_publication = observed.rejected_before_publication();
let result = observed.result;
#[cfg(test)]
if result.is_ok() {
rollback_fault_injection::after_rename(&dst_object, i)?;
}
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
duration_ms,
);
let position = if result.is_ok() {
let rank = successful_rename_completion_rank
.as_ref()
.map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1)
.unwrap_or(1);
if rank <= write_quorum {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST
} else {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL
}
} else {
rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR
};
rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms);
}
if rejected_before_publication {
dispatch_state = RenameDispatchState::RejectedBeforePublication;
}
result
})
.catch_unwind()
.await;
(dispatch_state, result)
}
});
let results = join_all(futures).await;
#[cfg(test)]
rollback_fault_injection::after_fanout(&fanout_dst_object);
(results, fanout_file_infos)
});
let mut disk_versions = vec![None; disk_count];
let mut data_dirs = vec![None; disk_count];
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let fanout_result = fanout.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
quorum_wait_started,
);
let (results, mut file_infos) = match fanout_result {
Ok(result) => result,
Err(_) => {
record_indeterminate_rename(disks, (&dst_bucket, &dst_object), rollback_receipt).await;
return Err(DiskError::Unexpected);
}
};
if rustfs_io_metrics::put_stage_metrics_enabled() {
let mut fanout_success = 0;
let mut fanout_error = 0;
let mut fanout_panic = 0;
for (_, result) in &results {
match result {
Ok(Ok(_)) => fanout_success += 1,
Ok(Err(_)) => fanout_error += 1,
Err(_) => fanout_panic += 1,
}
}
rustfs_io_metrics::record_put_rename_quorum_wait_fanout(
results.len(),
write_quorum,
fanout_success,
fanout_error,
fanout_panic,
);
}
let mut dispatch_states = Vec::with_capacity(results.len());
for (idx, (dispatch_state, result)) in results.iter().enumerate() {
dispatch_states.push(*dispatch_state);
match result {
Ok(Ok(res)) => {
data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir);
cleanup_data_dirs[idx] = res.cleanup_data_dir;
disk_versions[idx].clone_from(&res.sign);
old_current_sizes[idx] = res.old_current_size;
errs.push(None);
}
Ok(Err(e)) => {
errs.push(Some(e.clone()));
}
Err(_) => {
errs.push(Some(DiskError::Unexpected));
}
}
}
if issue3031_diag_enabled() {
let success_count = errs.iter().filter(|err| err.is_none()).count();
let failure_count = errs.len().saturating_sub(success_count);
let ignored_failure_count = errs
.iter()
.filter(|err| err.as_ref().is_some_and(|err| OBJECT_OP_IGNORED_ERRS.contains(err)))
.count();
let data_dir_vote_count = data_dirs.iter().filter(|data_dir| data_dir.is_some()).count();
let reduced_data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
warn!(
target: "rustfs_ecstore::set_disk",
src_bucket = %src_bucket,
src_object = %src_object,
dst_bucket = %dst_bucket,
dst_object = %dst_object,
write_quorum,
disk_count = errs.len(),
success_count,
failure_count,
ignored_failure_count,
data_dir_vote_count,
reduced_data_dir = ?reduced_data_dir,
errs = ?errs,
data_dirs = ?data_dirs,
"issue3031_rename_data_quorum_context"
);
}
if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
src_bucket = %src_bucket,
src_object = %src_object,
dst_bucket = %dst_bucket,
dst_object = %dst_object,
write_quorum,
ret_err = %ret_err,
errs = ?errs,
data_dirs = ?data_dirs,
"issue3031_rename_data_quorum_failed"
);
}
rollback_failed_rename(
disks,
file_infos,
&errs,
&dispatch_states,
&data_dirs,
(&dst_bucket, &dst_object),
RenameRollbackOwnership {
receipt: rollback_receipt,
namespace_commit_guard,
},
)
.await;
return Err(ret_err);
}
let mut backup_reclaims = Vec::new();
for (idx, disk) in disks.iter().enumerate() {
if errs[idx].is_some() {
continue;
}
let Some(rollback_dir) = data_dirs[idx] else {
continue;
};
if cleanup_data_dirs[idx] == Some(rollback_dir) {
continue;
}
let Some(disk) = disk.clone() else {
continue;
};
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
backup_reclaims.push(tokio::spawn(async move {
let backup_path = format!("{dst_object}/{rollback_dir}/{STORAGE_FORMAT_FILE_BACKUP}");
disk.delete(&dst_bucket, &backup_path, DeleteOptions::default()).await
}));
}
for result in join_all(backup_reclaims).await {
match result {
Ok(Ok(())) => {}
Ok(Err(DiskError::FileNotFound | DiskError::VolumeNotFound)) => {}
Ok(Err(err)) => {
warn!(
dst_bucket = %dst_bucket,
dst_object = %dst_object,
error = %err,
"rollback backup reclamation failed after committed rename"
);
}
Err(join_err) => {
warn!(
dst_bucket = %dst_bucket,
dst_object = %dst_object,
error = %join_err,
"rollback backup reclamation task failed after committed rename"
);
}
}
}
let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
let online_disks = Self::eval_disks(disks, &errs);
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
let committed_file_info = std::mem::take(&mut file_infos[committed_slot]);
let cleanup_disks = if let Some(data_dir) = data_dir {
disks
.iter()
.zip(errs.iter())
.zip(cleanup_data_dirs.iter())
.map(|((disk, err), old_data_dir)| {
if err.is_none() && *old_data_dir == Some(data_dir) {
disk.clone()
} else {
None
}
})
.collect()
} else {
vec![None; disks.len()]
};
Ok(RenameDataCommit {
capacity_disks: online_disks.clone(),
online_disks,
convergence,
data_dir,
cleanup_disks,
old_current_size,
committed_file_info,
tail_drain: None,
})
}
pub(in crate::set_disk) fn reduce_common_old_current_size(
old_current_sizes: &[Option<OldCurrentSize>],
write_quorum: usize,
) -> Option<OldCurrentSize> {
let mut counts: HashMap<OldCurrentSize, usize> = HashMap::new();
for observation in old_current_sizes.iter().flatten().copied() {
*counts.entry(observation).or_insert(0) += 1;
}
let mut max = 0;
let mut old_current_size = None;
for (observation, count) in counts {
if count > max {
max = count;
old_current_size = Some(observation);
}
}
if max >= write_quorum { old_current_size } else { None }
}
pub(in crate::set_disk) fn classify_rename_convergence(
disk_versions: &[Option<Vec<u8>>],
errs: &[Option<DiskError>],
) -> RenameConvergence {
if errs.iter().any(|err| err.is_some()) {
return RenameConvergence::PartialCommit;
}
let mut seen: Option<&Vec<u8>> = None;
let mut signed_count = 0usize;
let mut divergent = false;
for signature in disk_versions.iter() {
let Some(sig) = signature.as_ref() else {
continue;
};
signed_count += 1;
match seen {
None => seen = Some(sig),
Some(prev) if prev != sig => divergent = true,
Some(_) => {}
}
}
if divergent {
return RenameConvergence::SignatureDivergent;
}
if signed_count == 0 {
return RenameConvergence::Unknown;
}
if signed_count < disk_versions.len() {
return RenameConvergence::SignatureDivergent;
}
RenameConvergence::AllSuccessIdentical
}
#[tracing::instrument(level = "debug", skip(self, disks))]
pub(in crate::set_disk) async fn commit_rename_data_dir(
&self,
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
old_data_dir: &str,
committed_data_dir: &str,
write_quorum: usize,
) -> OldDataDirCleanup {
crate::hp_guard!("SetDisks::commit_rename_data_dir");
if old_data_dir.is_empty() || old_data_dir == committed_data_dir {
return OldDataDirCleanup::default();
}
let file_path = Arc::new(format!("{object}/{old_data_dir}"));
let bucket = Arc::new(bucket.to_string());
let attempted: Vec<bool> = disks.iter().map(|d| d.is_some()).collect();
let object_for_fault = Arc::new(object.to_string());
let futures = disks.iter().enumerate().map(|(idx, disk)| {
let file_path = file_path.clone();
let bucket = bucket.clone();
let disk = disk.clone();
let object_for_fault = object_for_fault.clone();
tokio::spawn(async move {
let Some(disk) = disk else {
return (false, Some(DiskError::DiskNotFound));
};
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&object_for_fault);
Self::rename_fanout_barrier(&object_for_fault, idx, rename_fanout_barrier_phase::CLEANUP).await;
if let Some(err) = Self::cleanup_injected_error(&object_for_fault, idx) {
return (false, Some(err));
}
match disk
.delete_data_dir(
&bucket,
&file_path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
{
Ok(DataDirDeleteStatus::Deleted) => (false, None),
Ok(DataDirDeleteStatus::Deferred) => (true, None),
Err(err) => (false, Some(err)),
}
})
});
let mut deferred = 0usize;
let errs: Vec<Option<DiskError>> = join_all(futures)
.await
.into_iter()
.map(|result| match result {
Ok((was_deferred, err)) => {
deferred += usize::from(was_deferred);
err
}
Err(join_err) => Some(DiskError::other(format!("old data dir cleanup task failed: {join_err}"))),
})
.collect();
let mut cleanup = classify_old_data_dir_cleanup(&errs, &attempted, write_quorum);
cleanup.deferred = deferred;
cleanup.reclaimed = cleanup.reclaimed.saturating_sub(deferred);
cleanup
}
pub(in crate::set_disk) async fn commit_rename_data_dir_and_mark_capacity(
&self,
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
old_data_dir: &str,
committed_data_dir: &str,
write_quorum: usize,
) -> OldDataDirCleanup {
let cleanup = self
.commit_rename_data_dir(disks, bucket, object, old_data_dir, committed_data_dir, write_quorum)
.await;
self.record_capacity_scope_if_needed(None, disks);
cleanup
}
#[cfg(test)]
fn cleanup_injected_error(object: &str, disk_index: usize) -> Option<DiskError> {
cleanup_fault_injection::injected_error(object, disk_index)
}
#[cfg(not(test))]
#[inline(always)]
fn cleanup_injected_error(_object: &str, _disk_index: usize) -> Option<DiskError> {
None
}
#[cfg(test)]
fn rename_injected_error(object: &str, disk_index: usize) -> Option<DiskError> {
rename_fault_injection::injected_error(object, disk_index)
}
#[cfg(not(test))]
#[inline(always)]
fn rename_injected_error(_object: &str, _disk_index: usize) -> Option<DiskError> {
None
}
#[cfg(test)]
#[inline]
fn record_read_version_call(object: &str, disk_index: usize) {
disk_call_counters::record(object, disk_call_counters::KIND_READ_VERSION, disk_index);
}
#[cfg(not(test))]
#[inline(always)]
fn record_read_version_call(_object: &str, _disk_index: usize) {}
#[cfg(test)]
#[inline]
fn record_metadata_slowtail_fault(object: &str, disk_index: usize) {
disk_call_counters::record(object, disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, disk_index);
}
#[cfg(not(test))]
#[inline(always)]
fn record_metadata_slowtail_fault(_object: &str, _disk_index: usize) {}
#[cfg(test)]
#[inline]
async fn read_version_fanout_barrier(object: &str, disk_index: usize) {
rename_fanout_barrier::checkpoint(object, disk_index, rename_fanout_barrier::PHASE_READ_VERSION).await;
}
#[cfg(test)]
#[inline]
async fn rename_fanout_barrier(object: &str, disk_index: usize, phase: &'static str) {
rename_fanout_barrier::checkpoint(object, disk_index, phase).await;
}
#[cfg(not(test))]
#[inline(always)]
#[allow(clippy::unused_async)]
async fn rename_fanout_barrier(_object: &str, _disk_index: usize, _phase: &'static str) {}
#[cfg(test)]
#[inline]
fn rename_fanout_task_guard(object: &str) -> rename_fanout_barrier::TaskGuard {
rename_fanout_barrier::task_guard(object)
}
#[cfg(not(test))]
#[inline(always)]
fn rename_fanout_task_guard(_object: &str) {}
pub(in crate::set_disk) async fn report_old_data_dir_cleanup(
&self,
bucket: &str,
object: &str,
old_dir: &str,
c: &OldDataDirCleanup,
) {
let actions = old_data_dir_cleanup_actions(c);
rustfs_io_metrics::record_old_data_dir_cleanup(c.attempted, c.reclaimed, c.unreclaimed_disks.len(), c.below_quorum);
if c.deferred > 0 {
debug!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket = %bucket,
object = %object,
old_data_dir = %old_dir,
deferred = c.deferred,
state = "old_data_cleanup_deferred",
"Old data directory cleanup deferred for active snapshot leases"
);
}
if actions.warn {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket = %bucket,
object = %object,
old_dir = %old_dir,
attempted = c.attempted,
reclaimed = c.reclaimed,
unreclaimed = ?c.unreclaimed_disks,
below_quorum = c.below_quorum,
"old data dir cleanup left residue after committed write"
);
}
if actions.enqueue_heal {
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(
rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(rustfs_heal_contracts::heal_channel::HealChannelPriority::Normal),
Some(self.pool_index),
Some(self.set_index),
),
)
.await;
}
}
#[tracing::instrument(skip(self))]
pub(in crate::set_disk) async fn cleanup_multipart_path(&self, paths: &[String]) {
if paths.is_empty() {
return;
}
let disks = self.get_disks_internal().await;
let mut errs = Vec::with_capacity(disks.len());
let processor = runtime_sources::batch_processors().write_processor();
let tasks: Vec<_> = disks
.iter()
.map(|disk| {
let disk = disk.clone();
let paths = paths.to_vec();
async move {
if let Some(disk) = disk {
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await
} else {
Err(DiskError::DiskNotFound)
}
}
})
.collect();
let results = processor.execute_batch(tasks).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
self.record_capacity_scope_if_needed(None, &disks);
if errs.iter().any(|e| e.is_some()) {
warn!("cleanup_multipart_path errs {:?}", &errs);
}
}
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
struct PartTransactionObservation {
transaction_meta: Option<Bytes>,
current_meta: Option<Bytes>,
rollback: bool,
err: Option<DiskError>,
}
enum PartTransactionOutcome {
Commit,
Rollback,
}
let disks = self.get_disks_internal().await;
let transaction_path = part_transaction_path(dst_object);
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
let current_meta_path = format!("{dst_object}.meta");
let reads = disks.iter().map(|disk| {
let disk = disk.clone();
let transaction_meta_path = transaction_meta_path.clone();
let rollback_path = rollback_path.clone();
let current_meta_path = current_meta_path.clone();
async move {
let Some(disk) = disk else {
return PartTransactionObservation {
transaction_meta: None,
current_meta: None,
rollback: false,
err: Some(DiskError::DiskNotFound),
};
};
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound) => None,
Err(err) => {
return PartTransactionObservation {
transaction_meta: None,
current_meta: None,
rollback: false,
err: Some(err),
};
}
};
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
Ok(_) => true,
Err(DiskError::FileNotFound) => false,
Err(err) => {
return PartTransactionObservation {
transaction_meta,
current_meta: None,
rollback: false,
err: Some(err),
};
}
};
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
Err(_) => None,
};
PartTransactionObservation {
transaction_meta,
current_meta,
rollback,
err: None,
}
}
});
let observations = join_all(reads).await;
let read_errs = observations
.iter()
.map(|observation| observation.err.clone())
.collect::<Vec<_>>();
if let Some(err) = reduce_write_quorum_errs(&read_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err);
}
if observations
.iter()
.filter(|observation| observation.err.is_none())
.all(|observation| observation.transaction_meta.is_none())
{
return Ok(false);
}
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
let mut transaction_meta_values = HashSet::new();
for observation in observations.iter().filter(|observation| observation.err.is_none()) {
if let Some(current) = &observation.current_meta {
*current_counts.entry(current.clone()).or_default() += 1;
}
if let Some(transaction_meta) = &observation.transaction_meta {
transaction_meta_values.insert(transaction_meta.clone());
}
}
let current_quorum = current_counts
.into_iter()
.find_map(|(meta, count)| (count >= write_quorum).then_some(meta));
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
let mut outcomes = Vec::with_capacity(observations.len());
for observation in &observations {
let outcome = if observation.err.is_none() && observation.transaction_meta.is_none() {
match &observation.current_meta {
Some(current_meta) if transaction_meta_values.contains(current_meta) => Some(PartTransactionOutcome::Commit),
_ => Some(PartTransactionOutcome::Rollback),
}
} else {
None
};
outcomes.push(outcome);
}
let decisions = observations
.iter()
.enumerate()
.filter_map(|(index, observation)| {
if observation.err.is_some() {
return None;
}
observation
.transaction_meta
.as_ref()
.map(|meta| (index, meta.clone(), observation.rollback))
})
.map(|(index, transaction_meta, rollback)| {
let disk = disks[index].clone();
let old_meta_path = old_meta_path.clone();
let old_meta_absent_path = old_meta_absent_path.clone();
let current_quorum = current_quorum.clone();
async move {
let Some(disk) = disk else {
return (index, Err(DiskError::DiskNotFound));
};
let result = async {
let action = if rollback {
PartTransactionAction::Rollback
} else if current_quorum.as_ref() == Some(&transaction_meta) {
PartTransactionAction::Commit
} else if let Some(current_quorum) = current_quorum {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
Ok(_) => PartTransactionAction::Commit,
Err(DiskError::FileNotFound) => {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
Ok(_) => PartTransactionAction::Commit,
Err(_) => return Err(DiskError::FileCorrupt),
}
}
Err(err) => return Err(err),
}
} else {
PartTransactionAction::Rollback
};
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
.await?;
let outcome = match action {
PartTransactionAction::Commit => PartTransactionOutcome::Commit,
PartTransactionAction::Rollback => PartTransactionOutcome::Rollback,
};
Ok(outcome)
};
(index, result.await)
}
});
let results = join_all(decisions).await;
let mut settle_errs = read_errs;
for result in results {
match result {
(index, Ok(outcome)) => outcomes[index] = Some(outcome),
(index, Err(err)) => settle_errs[index] = Some(err),
}
}
if let Some(err) = reduce_write_quorum_errs(&settle_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err);
}
let commit_count = outcomes
.iter()
.filter(|outcome| matches!(outcome, Some(PartTransactionOutcome::Commit)))
.count();
if commit_count >= write_quorum {
return Ok(true);
}
let rollback_count = outcomes
.iter()
.filter(|outcome| matches!(outcome, Some(PartTransactionOutcome::Rollback)))
.count();
if rollback_count >= write_quorum {
return Ok(false);
}
Err(DiskError::ErasureWriteQuorum)
}
pub(in crate::set_disk) async fn recover_part_transactions(
&self,
part_path: &str,
read_quorum: usize,
write_quorum: usize,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let listings = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
.await
}
}))
.await;
let mut transaction_parts = HashSet::new();
let mut errs = Vec::with_capacity(listings.len());
for listing in listings {
match listing {
Ok(entries) => {
errs.push(None);
for entry in entries {
let name = entry.trim_end_matches('/');
let Some(part_number) = name
.strip_prefix(".part.")
.and_then(|name| name.strip_suffix(".rustfs-txn"))
.and_then(|number| number.parse::<usize>().ok())
else {
continue;
};
transaction_parts.insert(part_number);
}
}
Err(err) => errs.push(Some(err)),
}
}
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
return Err(err);
}
for part_number in transaction_parts {
self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum)
.await?;
}
Ok(())
}
#[tracing::instrument(skip(disks, meta))]
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn rename_part(
&self,
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
meta: Bytes,
write_quorum: usize,
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
) -> disk::error::Result<Vec<Option<DiskStore>>> {
self.recover_part_transaction(dst_object, write_quorum).await?;
let part = ObjectPartInfo::unmarshal(&meta)?;
let integrity = part.integrity.map(Arc::new);
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let prepare_tasks = disks.iter().map(|disk| {
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
let meta = meta.clone();
let integrity = integrity.clone();
async move {
let disk = disk?;
let prepared = disk
.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await;
if let Err(error) = prepared {
return Some(Err(error));
}
if let Some(integrity) = integrity {
let Some((directory, _)) = dst_object.rsplit_once('/') else {
return Some(Err(DiskError::FileCorrupt));
};
let path = format!("{directory}/{}", integrity.file_name());
let result = async {
let mut reader = disk
.read_file_stream(&dst_bucket, &path, 0, rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE)
.await?;
let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
tokio::io::AsyncReadExt::read_exact(&mut reader, &mut header)
.await
.map_err(DiskError::from)?;
if header != integrity.index_header() {
return Err(DiskError::FileCorrupt);
}
Ok(())
}
.await;
return Some(result);
}
Some(Ok(()))
}
});
let prepare_results = join_all(prepare_tasks).await;
let prepare_errs = prepare_results
.into_iter()
.map(|result| match result {
Some(Ok(())) => None,
Some(Err(err)) => Some(err),
None => Some(DiskError::DiskNotFound),
})
.collect::<Vec<_>>();
let prepared_disks = Self::eval_disks(disks, &prepare_errs);
if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() {
self.recover_part_transaction(&dst_object, write_quorum).await?;
return Err(DiskError::ErasureWriteQuorum);
}
let mut errs = Vec::with_capacity(disks.len());
let futures = prepared_disks.iter().map(|disk| {
let disk = disk.clone();
let meta = meta.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await
} else {
Err(DiskError::DiskNotFound)
}
})
});
let results = join_all(futures).await;
for result in results {
match result? {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
if issue3031_diag_enabled() {
let success_count = errs.iter().filter(|err| err.is_none()).count();
let error_count = errs.len().saturating_sub(success_count);
let disk_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::DiskNotFound))).count();
let file_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::FileNotFound))).count();
warn!(
target: "rustfs_ecstore::set_disk",
src_bucket = %src_bucket,
src_object = %src_object,
dst_bucket = %dst_bucket,
dst_object = %dst_object,
write_quorum = write_quorum,
disk_count = errs.len(),
success_count = success_count,
error_count = error_count,
disk_not_found_count = disk_not_found_count,
file_not_found_count = file_not_found_count,
"issue3031_rename_part_context"
);
}
let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
if let Some(err) = reduced_err {
let rollbacks = prepared_disks.iter().filter_map(|disk| {
disk.clone().map(|disk| {
let dst_object = dst_object.clone();
async move {
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback)
.await
}
})
});
let rollback_results = join_all(rollbacks).await;
self.recover_part_transaction(&dst_object, write_quorum).await?;
if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) {
warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk");
}
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
return Err(err);
}
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
if !committed {
let err = DiskError::ErasureWriteQuorum;
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
return Err(err);
}
Ok(Self::eval_disks(&prepared_disks, &errs))
}
pub(in crate::set_disk) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
if disks.len() != errs.len() {
return Vec::new();
}
let mut online_disks = vec![None; disks.len()];
for (i, err_op) in errs.iter().enumerate() {
if err_op.is_none() {
online_disks[i].clone_from(&disks[i]);
}
}
online_disks
}
#[tracing::instrument(skip(disks, files))]
pub(in crate::set_disk) async fn write_unique_file_info(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
prefix: &str,
files: &[FileInfo],
write_quorum: usize,
) -> disk::error::Result<()> {
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for (i, disk) in disks.iter().enumerate() {
let mut file_info = files[i].clone();
file_info.erasure.index = i + 1;
futures.push(async move {
if let Some(disk) = disk {
disk.write_metadata(org_bucket, bucket, prefix, file_info).await
} else {
Err(DiskError::DiskNotFound)
}
});
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
let mut revert_futures = Vec::with_capacity(disks.len());
for (i, err) in errs.iter().enumerate() {
if err.is_some() {
continue;
}
if let Some(disk) = disks[i].as_ref() {
let mut rollback = FileInfo {
version_id: files[i].version_id,
..Default::default()
};
rollback.set_skip_tier_free_version();
revert_futures.push(async move {
if let Err(err) = disk
.delete_version(bucket, prefix, rollback, false, DeleteOptions::default())
.await
{
warn!("write meta revert err {:?}", err);
}
});
}
}
join_all(revert_futures).await;
return Err(err);
}
Ok(())
}
pub(in crate::set_disk) async fn update_object_meta(
&self,
bucket: &str,
object: &str,
fi: FileInfo,
disks: &[Option<DiskStore>],
) -> disk::error::Result<()> {
self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default())
.await
}
pub(in crate::set_disk) async fn update_object_meta_with_opts(
&self,
bucket: &str,
object: &str,
fi: FileInfo,
disks: &[Option<DiskStore>],
opts: &UpdateMetadataOpts,
) -> disk::error::Result<()> {
if fi.metadata.is_empty() && !opts.replace_user_metadata {
return Ok(());
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for disk in disks.iter() {
let fi = fi.clone();
futures.push(async move {
if let Some(disk) = disk {
disk.update_metadata(bucket, object, fi, opts).await
} else {
Err(DiskError::DiskNotFound)
}
})
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) {
return Err(err);
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
Ok(())
}
pub(in crate::set_disk) async fn delete_if_dangling(
&self,
bucket: &str,
object: &str,
meta_arr: &[FileInfo],
errs: &[Option<DiskError>],
data_errs_by_part: &HashMap<usize, Vec<usize>>,
opts: ObjectOptions,
) -> disk::error::Result<FileInfo> {
self.delete_if_dangling_with_proof(bucket, object, meta_arr, errs, data_errs_by_part, opts)
.await
.map(|(metadata, _)| metadata)
}
pub(in crate::set_disk) async fn delete_if_dangling_with_proof(
&self,
bucket: &str,
object: &str,
meta_arr: &[FileInfo],
errs: &[Option<DiskError>],
data_errs_by_part: &HashMap<usize, Vec<usize>>,
opts: ObjectOptions,
) -> disk::error::Result<(FileInfo, bool)> {
let (m, can_heal) = is_object_dangling(meta_arr, errs, data_errs_by_part);
if !can_heal {
return Err(DiskError::ErasureReadQuorum);
}
if file_info_is_valid_for_metadata(&m)
&& let Some(mod_time) = m.mod_time
{
let grace = dangling_delete_grace();
if !grace.is_zero() && OffsetDateTime::now_utc() - mod_time < grace {
let elapsed = OffsetDateTime::now_utc() - mod_time;
let retry_after_secs = grace.saturating_sub(elapsed).whole_seconds().max(0);
info!(
bucket = bucket,
object = object,
mod_time = %mod_time,
grace_secs = grace.whole_seconds(),
"skipping dangling-object deletion within grace window"
);
return Err(DiskError::dangling_delete_grace(retry_after_secs, grace.whole_seconds()));
}
}
let mut tags: HashMap<String, String> = HashMap::new();
tags.insert("set".to_string(), self.set_index.to_string());
tags.insert("pool".to_string(), self.pool_index.to_string());
tags.insert("merrs".to_string(), join_errs(errs));
tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}"));
if file_info_is_valid_for_metadata(&m) {
tags.insert("sz".to_string(), m.size.to_string());
tags.insert(
"mt".to_string(),
m.mod_time
.as_ref()
.map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()),
);
tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks));
} else {
tags.insert("invalid".to_string(), "1".to_string());
tags.insert(
"d:p".to_string(),
format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count),
);
}
let mut offline = 0;
for (i, err) in errs.iter().enumerate() {
let mut found = false;
if let Some(err) = err
&& err == &DiskError::DiskNotFound
{
found = true;
}
for p in data_errs_by_part {
if let Some(v) = p.1.get(i)
&& *v == CHECK_PART_DISK_NOT_FOUND
{
found = true;
break;
}
}
if found {
offline += 1;
}
}
if offline > 0 {
tags.insert("offline".to_string(), offline.to_string());
}
let mut fi = FileInfo::default();
if let Some(ref version_id) = opts.version_id {
fi.version_id = Uuid::parse_str(version_id).ok();
}
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
let disks = self.get_disks_internal().await;
let mut futures = Vec::with_capacity(disks.len());
for (disk_index, disk_op) in disks.iter().enumerate() {
#[cfg(not(test))]
let _ = disk_index;
let bucket = bucket.to_string();
let object = object.to_string();
let fi = fi.clone();
futures.push(async move {
#[cfg(test)]
if let Some(error) = crate::set_disk::ops::heal::injected_dangling_delete_error(&bucket, &object, disk_index) {
return Err(error);
}
if let Some(disk) = disk_op {
disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default())
.await
} else {
Err(DiskError::DiskNotFound)
}
});
}
let results = join_all(futures).await;
let mut all_deleted = !results.is_empty();
let mut delete_errs = Vec::with_capacity(results.len());
for (index, result) in results.into_iter().enumerate() {
let key = format!("ddisk-{index}");
let already_absent = matches!(
errs.get(index).and_then(Option::as_ref),
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
);
match result {
Ok(_) => {
tags.insert(key, "<nil>".to_string());
delete_errs.push(None);
}
Err(e) => {
all_deleted &= matches!(&e, DiskError::FileNotFound | DiskError::FileVersionNotFound);
tags.insert(key, e.to_string());
if already_absent || matches!(&e, DiskError::FileNotFound | DiskError::FileVersionNotFound) {
delete_errs.push(None);
} else {
delete_errs.push(Some(e));
}
}
}
}
let write_quorum = if file_info_is_valid_for_metadata(&m) {
m.write_quorum(self.default_write_quorum())
} else {
self.default_write_quorum()
};
if let Some(err) = reduce_write_quorum_errs(&delete_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err);
}
let absent = if all_deleted {
match Self::read_all_fileinfo(
&disks,
"",
bucket,
object,
opts.version_id.as_deref().unwrap_or(""),
false,
false,
false,
)
.await
{
Ok((_, after)) => after
.iter()
.all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))),
Err(_) => false,
}
} else {
false
};
Ok((m, absent))
}
fn reduce_delete_prefix_results(results: Vec<disk::error::Result<()>>, write_quorum: usize) -> disk::error::Result<()> {
let has_existing_volume = results
.iter()
.any(|result| matches!(result, Ok(()) | Err(DiskError::FileNotFound)));
let volume_not_found_count = results
.iter()
.filter(|result| matches!(result, Err(DiskError::VolumeNotFound)))
.count();
let errs = results
.into_iter()
.map(|result| result.err().filter(|err| !DiskError::is_err_object_not_found(err)))
.collect::<Vec<_>>();
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err);
}
if !has_existing_volume && volume_not_found_count >= write_quorum {
return Err(DiskError::VolumeNotFound);
}
Ok(())
}
#[cfg(test)]
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None, None)
.await
}
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
&self,
bucket: &str,
prefix: &str,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let write_quorum = disks.len() / 2 + 1;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
let mut futures = Vec::with_capacity(disks.len());
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
let disk_op = disk_op.clone();
let bucket = bucket.to_string();
let prefix = prefix.to_string();
let scanner_publication_commit_scope = scanner_publication_commit_scope.clone();
futures.push(async move {
if let Some(disk) = disk_op {
if let Some(scope) = scanner_publication_commit_scope.as_ref()
&& !scope.can_commit()
{
return Err(DiskError::other("scanner publication delete scope cannot commit"));
}
let external_guard = scanner_publication_commit_scope
.as_ref()
.map(|scope| Arc::new(scope.clone()) as Arc<dyn Send + Sync>);
disk.delete_with_scanner_publication_lease_and_guard(
&bucket,
&prefix,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
scanner_publication_lease_token,
external_guard,
)
.await
} else {
Err(DiskError::DiskNotFound)
}
});
}
run_scanner_publication_delete_owner(scanner_publication_commit_scope, move || async move {
Self::reduce_delete_prefix_results(join_all(futures).await, write_quorum)
})
.await
}
async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
let mut stack = vec![root.clone()];
let mut dirs: Vec<String> = Vec::new();
let mut committed_files: Vec<String> = Vec::new();
let mut blocked: HashSet<String> = HashSet::new();
let mut existed = false;
while let Some(dir) = stack.pop() {
let entries = match disk.list_dir("", bucket, &dir, 0).await {
Ok(entries) => entries,
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {
if dir == root {
return OrphanDirScan::Missing;
}
continue;
}
Err(_) => return OrphanDirScan::Unreadable,
};
existed = true;
let mut child_dirs = Vec::new();
let mut files = Vec::new();
let mut has_data = false;
for entry in entries {
if !is_safe_orphan_dir_entry(&entry) {
has_data = true;
break;
}
match entry.strip_suffix(SLASH_SEPARATOR) {
Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
None => files.push(entry),
}
}
if !has_data && !files.is_empty() {
let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
has_data = !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
}
if has_data {
block_orphan_dir_chain(&mut blocked, &root, &dir);
continue;
}
committed_files.extend(files.into_iter().map(|entry| format!("{dir}{SLASH_SEPARATOR}{entry}")));
dirs.push(dir);
stack.extend(child_dirs);
}
if existed {
OrphanDirScan::Scanned {
blocked,
dirs,
committed_files,
}
} else {
OrphanDirScan::Missing
}
}
async fn delete_purgeable_orphan_entries(
disk: &DiskStore,
bucket: &str,
object: &str,
mut empty_dirs: Vec<String>,
committed_files: Vec<String>,
) {
for delete_markers in [false, true] {
for file in &committed_files {
let is_marker = file.rsplit(SLASH_SEPARATOR).next().is_some_and(is_committed_delete_marker);
if is_marker != delete_markers {
continue;
}
if let Err(err) = disk
.delete(
bucket,
file,
DeleteOptions {
recursive: false,
immediate: true,
..Default::default()
},
)
.await
{
debug!(
event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
path = file,
error = ?err,
"Orphan prefix purge skipped"
);
return;
}
}
}
empty_dirs.reverse();
for dir in empty_dirs {
if let Err(err) = disk
.delete(
bucket,
&dir,
DeleteOptions {
recursive: false,
immediate: true,
..Default::default()
},
)
.await
{
debug!(
event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
path = dir,
error = ?err,
"Orphan prefix purge skipped"
);
}
}
}
pub(crate) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> disk::error::Result<bool> {
let backoff_key = format!("{bucket}{SLASH_SEPARATOR}{object}");
if self.orphan_purge_in_backoff(&backoff_key) {
return Ok(false);
}
let disks = self.get_disks_internal().await;
if disks.iter().any(Option::is_none) {
return Ok(false);
}
let mut per_disk: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
let mut blocked: HashSet<String> = HashSet::new();
for (i, disk) in disks.iter().enumerate() {
let Some(disk) = disk else { continue };
match Self::scan_orphan_dir(disk, bucket, object).await {
OrphanDirScan::Unreadable => return Ok(false),
OrphanDirScan::Scanned {
blocked: disk_blocked,
dirs,
committed_files,
} => {
blocked.extend(disk_blocked);
per_disk.push((i, dirs, committed_files));
}
OrphanDirScan::Missing => {}
}
}
if !blocked.is_empty() {
self.record_orphan_purge_backoff(backoff_key);
}
let mut purged = false;
for (i, dirs, committed_files) in per_disk {
let Some(disk) = disks[i].as_ref() else { continue };
let dirs = dirs.into_iter().filter(|dir| !blocked.contains(dir)).collect::<Vec<_>>();
let committed_files = committed_files
.into_iter()
.filter(|file| {
file.rsplit_once(SLASH_SEPARATOR)
.is_some_and(|(data_dir, _)| !blocked.contains(data_dir))
})
.collect::<Vec<_>>();
if dirs.is_empty() && committed_files.is_empty() {
continue;
}
purged = true;
Self::delete_purgeable_orphan_entries(disk, bucket, object, dirs, committed_files).await;
}
Ok(purged)
}
pub(crate) async fn purge_orphan_dir_objects_in_bucket(&self, bucket: &str) -> bool {
let disks = self.get_disks_internal().await;
if disks.is_empty() || disks.iter().any(Option::is_none) {
return false;
}
let mut prefixes = HashSet::new();
for disk in disks.iter().flatten() {
let entries = match disk.list_dir("", bucket, "", 0).await {
Ok(entries) => entries,
Err(_) => return false,
};
prefixes.extend(
entries
.into_iter()
.filter(|entry| entry.ends_with(SLASH_SEPARATOR) && is_safe_orphan_dir_entry(entry)),
);
}
let mut prefixes = prefixes.into_iter().collect::<Vec<_>>();
prefixes.sort_unstable();
let mut purged = false;
for prefix in prefixes {
match self.purge_orphan_dir_object(bucket, &prefix).await {
Ok(prefix_purged) => purged |= prefix_purged,
Err(_) => return false,
}
}
purged
}
pub(crate) async fn orphan_purge_has_complete_disk_set(&self) -> bool {
let disks = self.get_disks_internal().await;
!disks.is_empty() && disks.iter().all(Option::is_some)
}
fn orphan_purge_in_backoff(&self, key: &str) -> bool {
let backoff = self
.orphan_purge_backoff
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
backoff
.get(key)
.is_some_and(|scanned_at| scanned_at.elapsed() < ORPHAN_PURGE_BACKOFF)
}
fn record_orphan_purge_backoff(&self, key: String) {
let mut backoff = self
.orphan_purge_backoff
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let now = Instant::now();
backoff.retain(|_, scanned_at| now.duration_since(*scanned_at) < ORPHAN_PURGE_BACKOFF);
if backoff.len() >= ORPHAN_PURGE_BACKOFF_MAX_ENTRIES
&& let Some(oldest) = backoff
.iter()
.min_by_key(|(_, scanned_at)| **scanned_at)
.map(|(key, _)| key.clone())
{
backoff.remove(&oldest);
}
backoff.insert(key, now);
}
#[cfg(test)]
pub(crate) fn clear_orphan_purge_backoff(&self) {
self.orphan_purge_backoff
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clear();
}
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
}
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
}
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
let disks = self.get_disks_internal().await;
let mut referenced: HashSet<Uuid> = HashSet::new();
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
let mut healthy_metas = 0usize;
for (i, disk) in disks.iter().enumerate() {
let Some(disk) = disk else { continue };
let entries = match disk.list_dir("", bucket, object, 0).await {
Ok(entries) => entries,
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => continue,
Err(err) => return Err(err),
};
let mut physical = Vec::new();
for entry in &entries {
let Some(name) = entry.strip_suffix(SLASH_SEPARATOR) else { continue };
if let Ok(uuid) = Uuid::parse_str(name)
&& !uuid.is_nil()
{
physical.push(uuid);
}
}
let meta_path = path_join_buf(&[object, STORAGE_FORMAT_FILE]);
let buf = match disk.read_metadata(bucket, &meta_path).await {
Ok(buf) => buf,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {
if physical.is_empty() {
continue;
}
let mut committed = Vec::with_capacity(physical.len());
for dir in physical {
let data_dir = format!("{object}/{dir}");
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
entries.iter().any(|entry| {
entry
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
})
});
committed.push((dir, committed_delete));
}
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
per_disk_dirs.push((i, committed));
continue;
}
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object,
"reclaim_orphan_data_dirs: aborting, data dirs present without xl.meta on a disk"
);
return Ok(0);
}
Err(err) => return Err(err),
};
let meta = match FileMeta::load(&buf) {
Ok(meta) => meta,
Err(err) => {
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object, error = %err,
"reclaim_orphan_data_dirs: aborting, unparsable xl.meta on a disk"
);
return Ok(0);
}
};
match meta.get_data_dirs() {
Ok(dirs) => referenced.extend(dirs.into_iter().flatten().filter(|d| !d.is_nil())),
Err(err) => {
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object, error = %err,
"reclaim_orphan_data_dirs: aborting, could not decode data dirs from xl.meta"
);
return Ok(0);
}
}
healthy_metas += 1;
if !physical.is_empty() {
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
}
}
let mut removed = 0usize;
for (i, physical) in per_disk_dirs {
let Some(disk) = disks[i].as_ref() else { continue };
for (dir, committed_delete) in physical {
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
continue;
}
let stray = format!("{object}/{dir}");
if dry_run {
removed += 1;
debug!(
target: "rustfs_ecstore::set_disk",
event = "heal_abandoned_parts",
component = "ecstore",
subsystem = "heal",
state = "dry_run_matched",
result = "matched",
bucket, object, data_dir = %dir,
"Heal abandoned parts dry-run matched orphaned data directory"
);
continue;
}
match disk
.delete(
bucket,
&stray,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
{
Ok(()) => {
removed += 1;
info!(
target: "rustfs_ecstore::set_disk",
bucket, object, data_dir = %dir,
"reclaim_orphan_data_dirs: removed orphaned data directory"
);
}
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
Err(err) => {
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object, data_dir = %dir, error = %err,
"reclaim_orphan_data_dirs: failed to remove orphaned data directory"
);
}
}
}
}
Ok(removed)
}
fn write_precondition_lookup_error(
error: StorageError,
http_preconditions: &HTTPPreconditions,
bucket: &str,
object: &str,
) -> Option<StorageError> {
match error {
StorageError::VersionNotFound(_, _, _) | StorageError::ObjectNotFound(_, _) => {
if http_preconditions.if_match_value().is_some() {
Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()))
} else {
None
}
}
error => Some(error),
}
}
pub(in crate::set_disk) async fn check_write_precondition(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Option<StorageError> {
let mut lookup_opts = opts.clone();
let http_preconditions = lookup_opts.http_preconditions?;
lookup_opts.http_preconditions = None;
lookup_opts.no_lock = true;
let oi = self.get_object_info(bucket, object, &lookup_opts).await;
match oi {
Ok(oi) => {
if oi.delete_marker {
return (opts.data_movement || opts.preserve_delete_marker).then_some(StorageError::PreconditionFailed);
}
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
let if_match = http_preconditions.if_match_value().map(str::to_owned);
if should_prevent_write(&oi, if_none_match, if_match)
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
{
return Some(StorageError::PreconditionFailed);
}
}
Err(error) => {
return Self::write_precondition_lookup_error(error, &http_preconditions, bucket, object);
}
}
None
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub(in crate::set_disk) struct OldDataDirCleanup {
pub attempted: usize,
pub reclaimed: usize,
pub deferred: usize,
pub unreclaimed_disks: Vec<usize>,
pub below_quorum: bool,
}
impl OldDataDirCleanup {
pub fn has_residue(&self) -> bool {
!self.unreclaimed_disks.is_empty()
}
}
fn is_cleanup_not_found(e: &DiskError) -> bool {
matches!(e, DiskError::FileNotFound | DiskError::VolumeNotFound | DiskError::PathNotFound)
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
match joined {
Ok(res) => res,
Err(join_err) => Some(DiskError::other(format!("old data dir cleanup task failed: {join_err}"))),
}
}
fn classify_old_data_dir_cleanup(errs: &[Option<DiskError>], attempted: &[bool], write_quorum: usize) -> OldDataDirCleanup {
debug_assert_eq!(errs.len(), attempted.len());
let below_quorum = reduce_write_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some();
let mut reclaimed = 0usize;
let mut attempted_count = 0usize;
let mut unreclaimed_disks = Vec::new();
for (i, err) in errs.iter().enumerate() {
if !attempted.get(i).copied().unwrap_or(false) {
continue;
}
attempted_count += 1;
match err {
None => reclaimed += 1,
Some(e) if is_cleanup_not_found(e) => reclaimed += 1,
Some(_) => unreclaimed_disks.push(i),
}
}
OldDataDirCleanup {
attempted: attempted_count,
reclaimed,
deferred: 0,
unreclaimed_disks,
below_quorum,
}
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(in crate::set_disk) struct CleanupActions {
pub warn: bool,
pub emit_leak_metric: bool,
pub enqueue_heal: bool,
}
fn old_data_dir_cleanup_actions(c: &OldDataDirCleanup) -> CleanupActions {
CleanupActions {
warn: c.has_residue() || c.below_quorum,
emit_leak_metric: c.has_residue(),
enqueue_heal: c.has_residue(),
}
}
#[cfg(test)]
pub(in crate::set_disk) mod cleanup_fault_injection {
use super::DiskError;
use std::sync::{Mutex, OnceLock};
#[derive(Default)]
struct FaultState {
object: Option<String>,
fail_indices: Vec<usize>,
}
fn state() -> &'static Mutex<FaultState> {
static STATE: OnceLock<Mutex<FaultState>> = OnceLock::new();
STATE.get_or_init(|| Mutex::new(FaultState::default()))
}
pub struct FaultGuard;
impl Drop for FaultGuard {
fn drop(&mut self) {
if let Ok(mut s) = state().lock() {
*s = FaultState::default();
}
}
}
#[must_use]
pub fn fail_cleanup_on(object: &str, indices: &[usize]) -> FaultGuard {
let mut s = state().lock().expect("cleanup fault registry poisoned");
s.object = Some(object.to_string());
s.fail_indices = indices.to_vec();
FaultGuard
}
pub(super) fn injected_error(object: &str, disk_index: usize) -> Option<DiskError> {
let s = state().lock().expect("cleanup fault registry poisoned");
match &s.object {
Some(target) if target == object && s.fail_indices.contains(&disk_index) => {
Some(DiskError::other("injected old-data-dir cleanup failure (test-only)"))
}
_ => None,
}
}
}
#[cfg(test)]
pub(in crate::set_disk) mod rename_fault_injection {
use super::DiskError;
use std::sync::{Mutex, OnceLock};
#[derive(Default)]
struct FaultState {
object: Option<String>,
fail_indices: Vec<usize>,
}
fn state() -> &'static Mutex<FaultState> {
static STATE: OnceLock<Mutex<FaultState>> = OnceLock::new();
STATE.get_or_init(|| Mutex::new(FaultState::default()))
}
pub struct FaultGuard;
impl Drop for FaultGuard {
fn drop(&mut self) {
if let Ok(mut s) = state().lock() {
*s = FaultState::default();
}
}
}
#[must_use]
pub fn fail_rename_on(object: &str, indices: &[usize]) -> FaultGuard {
let mut s = state().lock().expect("rename fault registry poisoned");
s.object = Some(object.to_string());
s.fail_indices = indices.to_vec();
FaultGuard
}
pub(super) fn injected_error(object: &str, disk_index: usize) -> Option<DiskError> {
let s = state().lock().expect("rename fault registry poisoned");
match &s.object {
Some(target) if target == object && s.fail_indices.contains(&disk_index) => {
Some(DiskError::other("injected rename failure (test-only)"))
}
_ => None,
}
}
}
#[cfg(test)]
pub(in crate::set_disk) mod rollback_fault_injection {
use super::DiskError;
use std::{
collections::HashMap,
sync::{Mutex, OnceLock},
};
#[derive(Clone, Copy, Debug)]
pub(in crate::set_disk) enum Fault {
Io,
Panic,
IoAfterRename,
VolumeNotFoundAfterRename,
PanicAfterRename,
CoordinatorPanic,
RollbackCoordinatorPanic,
}
fn registry() -> &'static Mutex<HashMap<String, (usize, Fault)>> {
static REGISTRY: OnceLock<Mutex<HashMap<String, (usize, Fault)>>> = OnceLock::new();
REGISTRY.get_or_init(Mutex::default)
}
pub(in crate::set_disk) struct Guard(String);
impl Drop for Guard {
fn drop(&mut self) {
if let Ok(mut registry) = registry().lock() {
registry.remove(&self.0);
}
}
}
pub(in crate::set_disk) fn arm(object: &str, disk_index: usize, fault: Fault) -> Guard {
registry()
.lock()
.expect("rollback registry should not poison")
.insert(object.to_string(), (disk_index, fault));
Guard(object.to_string())
}
pub(super) fn before_undo(object: &str, disk_index: usize) -> Result<(), DiskError> {
let fault = registry()
.lock()
.expect("rollback registry should not poison")
.get(object)
.copied();
match fault {
Some((target, Fault::Io)) if target == disk_index => Err(DiskError::FaultyDisk),
Some((target, Fault::Panic)) if target == disk_index => panic!("injected rollback panic"),
_ => Ok(()),
}
}
pub(super) fn after_rename(object: &str, disk_index: usize) -> Result<(), DiskError> {
let fault = registry()
.lock()
.expect("rollback registry should not poison")
.get(object)
.copied();
match fault {
Some((target, Fault::IoAfterRename)) if target == disk_index => Err(DiskError::FaultyDisk),
Some((target, Fault::VolumeNotFoundAfterRename)) if target == disk_index => Err(DiskError::VolumeNotFound),
Some((target, Fault::PanicAfterRename)) if target == disk_index => panic!("injected panic after rename mutation"),
_ => Ok(()),
}
}
pub(super) fn after_fanout(object: &str) {
let fault = registry()
.lock()
.expect("rollback registry should not poison")
.get(object)
.copied();
if matches!(fault, Some((_, Fault::CoordinatorPanic))) {
panic!("injected rename coordinator panic");
}
}
pub(super) fn after_undo_dispatch(object: &str) {
let fault = registry()
.lock()
.expect("rollback registry should not poison")
.get(object)
.copied();
if matches!(fault, Some((_, Fault::RollbackCoordinatorPanic))) {
panic!("injected rollback coordinator panic");
}
}
}
#[cfg(test)]
pub(crate) mod disk_call_counters {
use std::collections::{HashMap, HashSet};
use std::sync::{Mutex, OnceLock};
pub const KIND_READ_VERSION: &str = "read_version";
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
pub const KIND_METADATA_SLOWTAIL_FAULT: &str = "metadata_slowtail_fault";
type CountKey = (String, String, usize);
#[derive(Default)]
struct Registry {
observed: HashSet<String>,
counts: HashMap<CountKey, u64>,
}
fn registry() -> &'static Mutex<Registry> {
static REG: OnceLock<Mutex<Registry>> = OnceLock::new();
REG.get_or_init(|| Mutex::new(Registry::default()))
}
pub(super) fn record(object: &str, kind: &str, disk_index: usize) {
let mut reg = registry().lock().expect("disk call-counter registry poisoned");
if !reg.observed.contains(object) {
return;
}
*reg.counts
.entry((object.to_string(), kind.to_string(), disk_index))
.or_insert(0) += 1;
}
#[must_use]
pub struct CallCounterScope {
object: String,
}
pub fn observe(object: &str) -> CallCounterScope {
let mut reg = registry().lock().expect("disk call-counter registry poisoned");
reg.observed.insert(object.to_string());
CallCounterScope {
object: object.to_string(),
}
}
impl CallCounterScope {
pub fn total(&self, kind: &str) -> u64 {
let reg = registry().lock().expect("disk call-counter registry poisoned");
reg.counts
.iter()
.filter(|((obj, k, _), _)| obj == &self.object && k == kind)
.map(|(_, count)| *count)
.sum()
}
pub fn for_disk(&self, kind: &str, disk_index: usize) -> u64 {
let reg = registry().lock().expect("disk call-counter registry poisoned");
reg.counts
.get(&(self.object.clone(), kind.to_string(), disk_index))
.copied()
.unwrap_or(0)
}
}
impl Drop for CallCounterScope {
fn drop(&mut self) {
let mut reg = registry().lock().expect("disk call-counter registry poisoned");
reg.observed.remove(&self.object);
reg.counts.retain(|(obj, _, _), _| obj != &self.object);
}
}
}
pub(in crate::set_disk) mod rename_fanout_barrier_phase {
pub const RENAME: &str = "rename";
pub const CLEANUP: &str = "cleanup";
pub const ROLLBACK: &str = "rollback";
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub const READ_VERSION: &str = "read_version";
}
#[cfg(test)]
pub(crate) mod rename_fanout_barrier {
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use tokio::sync::Notify;
pub use super::rename_fanout_barrier_phase::{
CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, ROLLBACK as PHASE_ROLLBACK,
};
pub const PHASE_NON_INLINE_HEDGE_TIMER: &str = "non_inline_hedge_timer";
struct Armed {
disk_index: usize,
phase: &'static str,
arrived: Arc<Notify>,
release: Arc<Notify>,
paused: Arc<AtomicBool>,
}
#[derive(Default)]
struct Registry {
armed: HashMap<String, Armed>,
observed: HashMap<String, Arc<AtomicUsize>>,
}
fn registry() -> &'static Mutex<Registry> {
static REG: OnceLock<Mutex<Registry>> = OnceLock::new();
REG.get_or_init(|| Mutex::new(Registry::default()))
}
fn lock() -> std::sync::MutexGuard<'static, Registry> {
registry().lock().expect("rename fan-out barrier registry poisoned")
}
#[must_use]
pub struct BarrierHandle {
object: String,
arrived: Arc<Notify>,
release: Arc<Notify>,
paused: Arc<AtomicBool>,
}
pub fn arm(object: &str, disk_index: usize, phase: &'static str) -> BarrierHandle {
let arrived = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let paused = Arc::new(AtomicBool::new(false));
lock().armed.insert(
object.to_string(),
Armed {
disk_index,
phase,
arrived: arrived.clone(),
release: release.clone(),
paused: paused.clone(),
},
);
BarrierHandle {
object: object.to_string(),
arrived,
release,
paused,
}
}
impl BarrierHandle {
pub async fn wait_until_paused(&self) {
if self.paused.load(Ordering::SeqCst) {
return;
}
self.arrived.notified().await;
}
pub fn is_paused(&self) -> bool {
self.paused.load(Ordering::SeqCst)
}
pub fn release(&self) {
self.release.notify_one();
}
}
impl Drop for BarrierHandle {
fn drop(&mut self) {
self.release.notify_one();
lock().armed.remove(&self.object);
}
}
pub(in crate::set_disk) async fn checkpoint(object: &str, disk_index: usize, phase: &'static str) {
let hooks = {
let reg = lock();
match reg.armed.get(object) {
Some(a) if a.disk_index == disk_index && a.phase == phase => {
Some((a.arrived.clone(), a.release.clone(), a.paused.clone()))
}
_ => None,
}
};
if let Some((arrived, release, paused)) = hooks {
paused.store(true, Ordering::SeqCst);
arrived.notify_one();
release.notified().await;
}
}
#[must_use]
pub struct TaskTrackerScope {
object: String,
}
pub fn observe_tasks(object: &str) -> TaskTrackerScope {
lock()
.observed
.entry(object.to_string())
.or_insert_with(|| Arc::new(AtomicUsize::new(0)));
TaskTrackerScope {
object: object.to_string(),
}
}
impl TaskTrackerScope {
pub fn running(&self) -> usize {
lock()
.observed
.get(&self.object)
.map(|c| c.load(Ordering::SeqCst))
.unwrap_or(0)
}
}
impl Drop for TaskTrackerScope {
fn drop(&mut self) {
lock().observed.remove(&self.object);
}
}
#[must_use]
pub struct TaskGuard {
counter: Option<Arc<AtomicUsize>>,
}
pub(in crate::set_disk) fn task_guard(object: &str) -> TaskGuard {
let counter = lock().observed.get(object).cloned();
if let Some(c) = &counter {
c.fetch_add(1, Ordering::SeqCst);
}
TaskGuard { counter }
}
impl Drop for TaskGuard {
fn drop(&mut self) {
if let Some(c) = &self.counter {
c.fetch_sub(1, Ordering::SeqCst);
}
}
}
}
#[cfg(test)]
mod tests {
use crate::disk::error::HEAL_DANGLING_DELETE_GRACE_MESSAGE;
use crate::disk::local::{DurabilityMode, durability_mode_override};
use super::*;
use std::io::Cursor;
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[test]
fn orphan_dir_entries_must_be_single_relative_components() {
for entry in ["part.1", "child/", "delete-data.00000000-0000-0000-0000-000000000001"] {
assert!(is_safe_orphan_dir_entry(entry), "{entry:?} should be accepted");
}
for entry in ["", "/", ".", "..", "../", "child//", "a/b", r"a\b", "./"] {
assert!(!is_safe_orphan_dir_entry(entry), "{entry:?} should be rejected");
}
}
#[test]
#[serial_test::serial(codec_streaming_env)]
fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("on")),
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", Some("true")),
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", Some("true")),
],
|| assert!(!non_inline_data_read_early_stop_allowed(true, "bucket", "object")),
);
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("off")),
],
|| assert!(non_inline_data_read_early_stop_allowed(true, "bucket", "object")),
);
}
#[tokio::test]
async fn scanner_delete_owner_survives_waiter_cancellation() {
let movement_gate = Arc::new(tokio::sync::RwLock::new(()));
let movement_permit = movement_gate.clone().read_owned().await;
let scope = crate::object_api::ScannerPublicationCommitScope::new_storage_owned(
7,
tokio::time::Instant::now() + std::time::Duration::from_secs(30),
Vec::new(),
movement_permit,
);
scope.try_begin().expect("delete scope should enter flight");
let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move {
started_tx.send(()).expect("delete owner should start");
release_rx.await.expect("delete owner should be released");
finished_tx.send(()).expect("delete owner should finish");
Ok(())
}));
started_rx.await.expect("delete owner should run");
drop(scope_guard);
waiter.abort();
assert_eq!(
scope.state(),
crate::object_api::ScannerPublicationCommitState::InFlight,
"caller cancellation must not classify an owned delete as indeterminate"
);
let mut movement_writer = Box::pin(movement_gate.write_owned());
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut movement_writer)
.await
.is_err(),
"movement transition must remain fenced while delete owner drains"
);
release_tx.send(()).expect("delete owner should remain alive");
finished_rx.await.expect("delete owner should drain");
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
if scope.state() == crate::object_api::ScannerPublicationCommitState::Committed {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("delete owner should report a terminal result");
assert!(
scope.release_movement_permit().await,
"terminal delete should release its movement permit"
);
movement_writer.await;
}
#[test]
fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
let create_only = HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
};
let replace_only = HTTPPreconditions {
if_match: Some("etag".to_string()),
..Default::default()
};
assert!(matches!(
SetDisks::write_precondition_lookup_error(StorageError::ErasureReadQuorum, &create_only, "bucket", "object",),
Some(StorageError::ErasureReadQuorum)
));
assert!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&create_only,
"bucket",
"object",
)
.is_none()
);
assert!(matches!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&replace_only,
"bucket",
"object",
),
Some(StorageError::ObjectNotFound(_, _))
));
}
fn metadata_test_fileinfo(object: &str) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = "bucket".to_string();
fi.name = object.to_string();
fi.size = 1;
fi.erasure.index = 1;
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
fi
}
fn metadata_test_delete_marker(object: &str, version_id: Uuid, mod_time: OffsetDateTime) -> FileInfo {
FileInfo {
volume: "bucket".to_string(),
name: object.to_string(),
version_id: Some(version_id),
deleted: true,
mod_time: Some(mod_time),
..Default::default()
}
}
fn read_part_test_part(number: usize, etag: &str) -> ObjectPartInfo {
ObjectPartInfo {
number,
etag: etag.to_string(),
..Default::default()
}
}
fn read_part_test_error(number: usize, error: &str) -> ObjectPartInfo {
ObjectPartInfo {
number,
error: Some(error.to_string()),
..Default::default()
}
}
async fn read_multiple_test_disk(bucket: &str, objects: &[(&str, &[u8])]) -> (TempDir, DiskStore) {
let dir = tempfile::tempdir().expect("tempdir should be created");
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("disk should be created");
match disk.make_volume(bucket).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("bucket should be available: {err:?}"),
}
for (object, body) in objects {
disk.write_all(bucket, object, Bytes::copy_from_slice(body))
.await
.expect("object should be written");
}
(dir, disk)
}
#[tokio::test]
async fn orphan_cleanup_preserves_object_published_after_scan() {
let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
let transaction = Uuid::new_v4();
let residue = dir
.path()
.join("bucket")
.join("pfx")
.join("object")
.join(Uuid::new_v4().to_string());
tokio::fs::create_dir_all(&residue)
.await
.expect("committed data directory should be created");
tokio::fs::write(residue.join("part.1"), b"stale")
.await
.expect("stale part should be written");
tokio::fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")), [])
.await
.expect("committed delete marker should be written");
let OrphanDirScan::Scanned {
blocked,
dirs: empty_dirs,
committed_files,
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
else {
panic!("committed residue should be classified as purgeable");
};
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
let nested_object = residue.join("nested");
tokio::fs::create_dir_all(&nested_object)
.await
.expect("concurrent object directory should be created");
tokio::fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"new metadata")
.await
.expect("concurrent object metadata should be written");
SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
assert!(
nested_object.join(STORAGE_FORMAT_FILE).exists(),
"an object published after classification must survive cleanup"
);
assert!(!residue.join("part.1").exists(), "classified stale data should be reclaimed");
}
#[cfg(unix)]
#[tokio::test]
async fn orphan_cleanup_keeps_commit_marker_when_residue_delete_fails() {
use std::os::unix::fs::PermissionsExt;
let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4());
let residue = dir
.path()
.join("bucket")
.join("pfx")
.join("object")
.join(Uuid::new_v4().to_string());
tokio::fs::create_dir_all(&residue)
.await
.expect("committed data directory should be created");
tokio::fs::write(residue.join("part.1"), b"stale")
.await
.expect("stale part should be written");
tokio::fs::write(residue.join(&marker_name), [])
.await
.expect("committed delete marker should be written");
let OrphanDirScan::Scanned {
blocked,
dirs: empty_dirs,
committed_files,
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
else {
panic!("committed residue should be classified as purgeable");
};
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
.await
.expect("residue directory should become read-only");
SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
let part_remains = residue.join("part.1").exists();
let marker_remains = residue.join(marker_name).exists();
tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o755))
.await
.expect("residue directory permissions should be restored");
assert!(part_remains, "the injected residue delete failure should retain the part");
assert!(marker_remains, "the commit marker must remain so a later cleanup can retry");
}
async fn io_primitives_test_set(disks: Vec<Option<DiskStore>>, default_parity_count: usize) -> Arc<SetDisks> {
let set_drive_count = disks.len();
SetDisks::new(
"io-primitives-test".to_string(),
Arc::new(RwLock::new(disks)),
set_drive_count,
default_parity_count,
0,
0,
Vec::new(),
FormatV3::new(1, 1),
Vec::new(),
)
.await
}
async fn write_raw_file_meta_unchecked(disk: &DiskStore, bucket: &str, object: &str, metadata: FileMeta) {
let encoded = metadata.marshal_msg().expect("raw regression metadata should serialize");
disk.write_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"), Bytes::from(encoded))
.await
.expect("raw regression metadata should be installed");
}
async fn write_raw_file_info_unchecked(disk: &DiskStore, bucket: &str, object: &str, file_info: FileInfo) {
let mut metadata = FileMeta::new();
metadata
.add_version(file_info)
.expect("raw regression metadata should encode");
write_raw_file_meta_unchecked(disk, bucket, object, metadata).await;
}
fn failed_read_repair_submitter(
_request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
) -> ReadRepairAdmissionFuture {
Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) })
}
fn accepted_read_repair_submitter(
_request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
) -> ReadRepairAdmissionFuture {
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
}
fn dropped_read_repair_submitter(
_request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
) -> ReadRepairAdmissionFuture {
Box::pin(async {
ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Dropped(
rustfs_heal_contracts::heal_channel::HealAdmissionDropReason::PolicyDropped,
))
})
}
fn test_object_bitrot_reader() -> ObjectBitrotReader {
BitrotReader::new(
ShardReader::Stream(Box::new(Cursor::new(vec![1u8, 2, 3, 4]))),
4,
HashAlgorithm::None,
false,
)
}
fn test_deferred_object_bitrot_reader() -> (ObjectBitrotReader, DeferredReaderStripeHandle) {
create_deferred_bitrot_reader_with_stripe_handle(
Some(Bytes::from_static(&[1, 2, 3, 4])),
None,
"bucket",
"object/part.1",
0,
4,
4,
HashAlgorithm::None,
false,
false,
)
}
#[test]
fn dangling_delete_grace_defaults_to_one_hour() {
temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, None::<&str>, || {
assert_eq!(dangling_delete_grace(), time::Duration::seconds(3600));
});
}
#[test]
fn dangling_delete_grace_env_override_and_disable() {
temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, Some("120"), || {
assert_eq!(dangling_delete_grace(), time::Duration::seconds(120));
});
temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, Some("0"), || {
assert!(dangling_delete_grace().is_zero());
});
}
async fn call_counter_local_disks(bucket: &str, count: usize) -> (Vec<TempDir>, Vec<Option<DiskStore>>) {
let mut dirs = Vec::with_capacity(count);
let mut disks = Vec::with_capacity(count);
for _ in 0..count {
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
dirs.push(dir);
disks.push(Some(disk));
}
(dirs, disks)
}
async fn reopen_local_disk(dir: &TempDir) -> DiskStore {
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("disk should reopen from the same tempdir")
}
async fn prepare_rename_source_dirs(dirs: &[TempDir], disks: &[Option<DiskStore>], source: &str) {
for (dir, disk) in dirs.iter().zip(disks.iter()) {
let Some(disk) = disk else {
continue;
};
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
}
std::fs::create_dir_all(dir.path().join(RUSTFS_META_TMP_BUCKET).join(source))
.expect("rename staging source dir should be created");
}
}
#[test]
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
|| {
assert_eq!(
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
Some(Duration::from_millis(25))
);
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
},
);
}
#[test]
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
],
|| {
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
},
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-fault-bucket";
let object = "objects/metadata-slowtail-fault-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let calls = disk_call_counters::observe(object);
let read_without_data =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
let (_, _, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_without_data)
.await
.expect("non-data metadata fanout should complete")
.expect("metadata fanout without read_data should resolve");
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(
calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT),
0,
"non-data metadata fanout must not apply the data-read slowtail hook"
);
let read_with_data =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
.await
.expect("injected slowtail should eventually complete")
.expect("data-read metadata fanout should resolve");
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(
calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT),
1,
"exactly one data-read metadata task should apply the slowtail hook"
);
assert_eq!(
calls.for_disk(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, 3),
1,
"the configured slow disk should apply the slowtail hook exactly once"
);
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-early-stop-bucket";
let object = "objects/metadata-slowtail-early-stop-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
&disks, bucket, bucket, object, "", true, false, false, true, 2,
));
assert!(
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
.await
.is_err(),
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
.await
.expect("injected early-stop slowtail should eventually complete")
.expect("early-stop metadata fanout should resolve");
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(diagnostics.total_responses(), DISKS);
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_gate_stops_before_unneeded_tail() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-gated-bucket";
let object = "objects/metadata-slowtail-gated-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await;
let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2);
let slow_disk = *order.get(3).expect("four-disk fanout should have a deferred tail disk");
let slow_disk_env = slow_disk.to_string();
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let calls = disk_call_counters::observe(object);
let read_with_data =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(500), read_with_data)
.await
.expect("gated metadata read should stop before the deferred slow tail")
.expect("gated metadata fanout should resolve");
assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3);
assert!(errs.iter().all(Option::is_none));
assert!(diagnostics.total_responses() < DISKS);
assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 0);
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_gate_hedges_an_initial_slow_data_shard() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-gated-initial-bucket";
let object = "objects/metadata-slowtail-gated-initial-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await;
let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2);
let slow_disk = *order.get(1).expect("four-disk fanout should have an initial data disk");
let spare_disk = *order.get(3).expect("four-disk fanout should have a spare disk");
let slow_disk_env = slow_disk.to_string();
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("500")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let calls = disk_call_counters::observe(object);
let read_with_data =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(300), read_with_data)
.await
.expect("gated metadata read should hedge the initial slow shard")
.expect("gated metadata fanout should resolve");
assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3);
assert!(errs.iter().all(Option::is_none));
assert!(diagnostics.total_responses() < DISKS);
assert_eq!(calls.for_disk(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, slow_disk), 1);
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, spare_disk), 1);
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_version_call_counter_observes_spawned_fanout() {
const DISKS: usize = 4;
let bucket = "counter-bucket";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
let object = "counter-object";
let scope = disk_call_counters::observe(object);
let _ = SetDisks::read_all_fileinfo(&disks, bucket, bucket, object, "", false, false, false).await;
assert_eq!(
scope.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"every online disk must record exactly one read_version, even from a spawned task"
);
for idx in 0..DISKS {
assert_eq!(
scope.for_disk(disk_call_counters::KIND_READ_VERSION, idx),
1,
"disk {idx} should record exactly one read_version"
);
}
let _ = SetDisks::read_all_fileinfo(&disks, bucket, bucket, object, "", false, false, false).await;
assert_eq!(scope.total(disk_call_counters::KIND_READ_VERSION), (DISKS * 2) as u64);
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn metadata_read_version_coalescer_bypasses_local_disks() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-local-bypass-bucket";
let object_a = "coalesced-local-object-a";
let object_b = "coalesced-local-object-b";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
],
async {
let calls = disk_call_counters::observe(object_a);
let disks_a = disks.clone();
let disks_b = disks.clone();
let read_a = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_a, "", bucket, object_a, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
tokio::task::yield_now().await;
let read_b = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_b, "", bucket, object_b, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
let (metadata_a, errs_a) = read_a
.await
.expect("first read task should not panic")
.expect("first coalesced read should resolve");
let (metadata_b, errs_b) = read_b
.await
.expect("second read task should not panic")
.expect("second coalesced read should resolve");
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
assert!(errs_a.iter().all(Option::is_none));
assert!(errs_b.iter().all(Option::is_none));
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"local disks still execute the ordinary per-disk read_version path"
);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"GET coalescing targets internode RPC count only and must not batch local disk reads"
);
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn metadata_read_version_coalescer_requires_get_object_intent() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-default-bypass-bucket";
let object = "default-bypass-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
let calls = disk_call_counters::observe(object);
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
.await
.expect("default metadata read should resolve");
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
);
})
.await;
drop(dirs);
}
#[test]
fn batch_read_version_response_mapping_preserves_index_and_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-c".to_string(),
version_id: "v-c".to_string(),
},
];
let ok_file_info = FileInfo {
name: "object-a".to_string(),
..Default::default()
};
let responses = vec![
BatchReadVersionResp {
index: 2,
path: "object-c".to_string(),
version_id: "v-c".to_string(),
success: false,
file_info: FileInfo::default(),
error: "disk read failed".to_string(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: ok_file_info,
error: String::new(),
error_code: 0,
},
];
let expected_items = expected_batch_read_version_items(&expected_items);
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
.expect("slot 0 should exist")
.expect("slot 0 should map the success response by index");
assert_eq!(first.name, "object-a");
let missing = results
.next()
.expect("slot 1 should exist")
.expect_err("slot 1 should stay missing");
assert!(
missing.to_string().contains("response missing"),
"unexpected missing response error: {missing}"
);
let failed = results
.next()
.expect("slot 2 should exist")
.expect_err("slot 2 should map the response error");
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
assert!(results.next().is_none());
}
#[test]
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
];
let expected_items = expected_batch_read_version_items(&expected_items);
let results = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileNotFound.to_string(),
error_code: DiskError::FileNotFound.to_u32(),
},
BatchReadVersionResp {
index: 1,
path: "object-b".to_string(),
version_id: "v-b".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileVersionNotFound.to_string(),
error_code: DiskError::FileVersionNotFound.to_u32(),
},
],
);
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
assert!(matches!(
results.get(1).expect("slot 1 should exist"),
Err(DiskError::FileVersionNotFound)
));
}
#[test]
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
let expected_items = vec![BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let expected_items = expected_batch_read_version_items(&expected_items);
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
index: 0,
path: "object-b".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-b".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
}],
)
.pop()
.expect("slot 0 should exist")
.expect_err("identity mismatch should fail closed");
assert!(
mismatched.to_string().contains("identity mismatch"),
"unexpected mismatch error: {mismatched}"
);
let duplicate = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
],
)
.pop()
.expect("slot 0 should exist")
.expect_err("duplicate response index should fail closed");
assert!(
duplicate.to_string().contains("duplicate index"),
"unexpected duplicate error: {duplicate}"
);
}
fn expected_batch_read_version_items(items: &[BatchReadVersionItem]) -> Vec<ExpectedBatchReadVersionItem> {
items.iter().map(ExpectedBatchReadVersionItem::from).collect()
}
#[tokio::test]
async fn call_counter_isolates_unobserved_objects_and_clears_on_drop() {
let bucket = "counter-bucket-iso";
let (dirs, disks) = call_counter_local_disks(bucket, 1).await;
let object = "iso-object";
let _ = SetDisks::read_all_fileinfo(&disks, bucket, bucket, object, "", false, false, false).await;
{
let scope = disk_call_counters::observe(object);
assert_eq!(
scope.total(disk_call_counters::KIND_READ_VERSION),
0,
"reads before the scope existed must not be counted"
);
let _ = SetDisks::read_all_fileinfo(&disks, bucket, bucket, object, "", false, false, false).await;
assert_eq!(scope.total(disk_call_counters::KIND_READ_VERSION), 1);
}
let scope = disk_call_counters::observe(object);
assert_eq!(
scope.total(disk_call_counters::KIND_READ_VERSION),
0,
"dropping a scope must clear its counts"
);
drop(dirs);
}
fn valid_metadata_fanout_fileinfo(
bucket: &str,
object: &str,
version_id: Uuid,
data_dir: Uuid,
mod_time: OffsetDateTime,
) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = bucket.to_string();
fi.name = object.to_string();
fi.size = 1;
fi.erasure.index = 1;
fi.version_id = Some(version_id);
fi.is_latest = true;
fi.data_dir = Some(data_dir);
fi.mod_time = Some(mod_time);
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
fi.add_object_part(1, "part-etag".to_string(), 1, fi.mod_time, 1, None, None);
fi
}
async fn install_metadata_fanout_fileinfo(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
missing_part_disk: Option<usize>,
) {
let version_id = Uuid::new_v4();
let data_dir = Uuid::new_v4();
let mod_time = OffsetDateTime::now_utc();
for (index, disk) in disks
.iter()
.enumerate()
.filter_map(|(index, disk)| disk.as_ref().map(|disk| (index, disk)))
{
if missing_part_disk != Some(index) {
disk.write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from_static(b"x"))
.await
.expect("part data should be installed on every disk");
}
disk.write_metadata(
bucket,
bucket,
object,
valid_metadata_fanout_fileinfo(bucket, object, version_id, data_dir, mod_time),
)
.await
.expect("metadata should be installed on every disk");
}
}
async fn install_mapped_metadata_fanout_fileinfo(disks: &[Option<DiskStore>], bucket: &str, object: &str) {
let version_id = Uuid::new_v4();
let data_dir = Uuid::new_v4();
let mod_time = OffsetDateTime::now_utc();
let distribution = FileInfo::new(&metadata_distribution_key(bucket, object), 2, 2)
.erasure
.distribution;
for (index, disk) in disks
.iter()
.enumerate()
.filter_map(|(index, disk)| disk.as_ref().map(|disk| (index, disk)))
{
disk.write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from_static(b"x"))
.await
.expect("part data should be installed on every disk");
let mut file_info = valid_metadata_fanout_fileinfo(bucket, object, version_id, data_dir, mod_time);
file_info.erasure.distribution = distribution.clone();
file_info.erasure.index = *distribution
.get(index)
.expect("mapped metadata distribution should cover every disk");
disk.write_metadata(bucket, bucket, object, file_info)
.await
.expect("mapped metadata should be installed on every disk");
}
}
async fn inline_metadata_fanout_fileinfos_with_mode(
bucket: &str,
object: &str,
payload: &[u8],
uses_legacy_checksum: bool,
) -> Vec<FileInfo> {
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await
}
async fn inline_metadata_fanout_fileinfos_with_geometry(
bucket: &str,
object: &str,
payload: &[u8],
uses_legacy_checksum: bool,
data_shards: usize,
parity_shards: usize,
) -> Vec<FileInfo> {
let distribution_key = metadata_distribution_key(bucket, object);
let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards);
base.volume = bucket.to_string();
base.name = object.to_string();
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
base.is_latest = true;
base.version_id = Some(Uuid::new_v4());
base.data_dir = Some(Uuid::new_v4());
base.mod_time = Some(OffsetDateTime::now_utc());
base.metadata.insert("etag".to_string(), "etag-inline".to_string());
base.add_object_part(1, "part-etag-inline".to_string(), payload.len(), base.mod_time, base.size, None, None);
base.set_inline_data();
base.uses_legacy_checksum = uses_legacy_checksum;
let erasure = coding::Erasure::new_with_options(
base.erasure.data_blocks,
base.erasure.parity_blocks,
base.erasure.block_size,
base.uses_legacy_checksum,
);
let shards = erasure.encode_data(payload).expect("inline payload should encode");
let checksum_algo = if base.uses_legacy_checksum {
HashAlgorithm::HighwayHash256SLegacy
} else {
HashAlgorithm::HighwayHash256S
};
let mut files = Vec::with_capacity(shards.len());
for (index, shard) in shards.into_iter().enumerate() {
let mut writer = coding::BitrotWriterWrapper::new(
coding::CustomWriter::new_inline_buffer(),
erasure.shard_size(),
checksum_algo.clone(),
);
writer.write(&shard).await.expect("inline shard should write");
writer.shutdown().await.expect("inline writer should shutdown");
let mut fi = base.clone();
fi.erasure.index = index + 1;
fi.data = Some(Bytes::from(
writer
.into_inline_data()
.expect("inline bitrot writer should retain encoded data"),
));
files.push(fi);
}
files
}
async fn inline_metadata_fanout_fileinfos(bucket: &str, object: &str, payload: &[u8]) -> Vec<FileInfo> {
inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await
}
async fn install_inline_metadata_fanout_fileinfo(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
payload: &[u8],
mutate: impl FnOnce(&mut [FileInfo]),
) {
let mut files = inline_metadata_fanout_fileinfos(bucket, object, payload).await;
mutate(&mut files);
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
}
async fn install_inline_metadata_fanout_fileinfo_with_geometry(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
payload: &[u8],
data_shards: usize,
parity_shards: usize,
mutate: impl FnOnce(&mut [FileInfo]),
) {
let mut files =
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await;
mutate(&mut files);
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
}
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
let distribution = files
.first()
.map(|file| file.erasure.distribution.clone())
.expect("inline metadata fixture should include shards");
for (disk_index, disk) in disks
.iter()
.enumerate()
.filter_map(|(disk_index, disk)| disk.as_ref().map(|disk| (disk_index, disk)))
{
let block_index = distribution
.get(disk_index)
.copied()
.expect("inline metadata fixture should cover every disk");
let file_info = files
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
.expect("inline metadata fixture should include every distributed shard")
.clone();
disk.write_metadata(bucket, bucket, object, file_info)
.await
.expect("inline metadata should be installed on every disk");
}
}
fn object_with_initial_data_shards(bucket: &str, prefix: &str, data_shards: usize, initial_fanout: usize) -> String {
(0..1000)
.map(|index| format!("{prefix}-{index}"))
.find(|name| {
let order = bounded_metadata_fanout_order(bucket, name, data_shards + 2, 2);
let distribution_key = metadata_distribution_key(bucket, name);
let distribution = FileInfo::new(&distribution_key, data_shards, 2).erasure.distribution;
order
.iter()
.take(initial_fanout)
.filter_map(|disk_index| distribution.get(*disk_index).copied())
.filter(|block_index| (1..=data_shards).contains(block_index))
.collect::<HashSet<_>>()
.len()
== data_shards
})
.expect("test should find an object whose initial fanout covers every data shard")
}
fn initial_data_shard_indexes(bucket: &str, object: &str, data_shards: usize, initial_fanout: usize) -> Vec<usize> {
let order = bounded_metadata_fanout_order(bucket, object, data_shards + 2, 2);
let distribution_key = metadata_distribution_key(bucket, object);
let distribution = FileInfo::new(&distribution_key, data_shards, 2).erasure.distribution;
order
.iter()
.take(initial_fanout)
.filter_map(|disk_index| distribution.get(*disk_index).copied())
.filter(|block_index| (1..=data_shards).contains(block_index))
.collect()
}
fn bounded_spare_disk_index(bucket: &str, object: &str, data_shards: usize, parity_shards: usize) -> usize {
let total_disks = data_shards + parity_shards;
let initial_fanout = if data_shards == parity_shards {
data_shards + 1
} else {
data_shards
};
*bounded_metadata_fanout_order(bucket, object, total_disks, parity_shards)
.get(initial_fanout)
.expect("test geometry should leave one bounded spare disk")
}
#[test]
fn bounded_metadata_fanout_order_prioritizes_default_data_shards() {
let bucket = "bounded-order-bucket";
let object = "bounded-order-data-shards-first";
let order = bounded_metadata_fanout_order(bucket, object, 16, 4);
let distribution_key = [bucket, object].join("/");
let distribution = FileInfo::new(&distribution_key, 12, 4).erasure.distribution;
let initial_blocks: HashSet<_> = order
.iter()
.take(12)
.filter_map(|disk_index| distribution.get(*disk_index).copied())
.collect();
assert_eq!(order.len(), 16);
assert_eq!(order.iter().copied().collect::<HashSet<_>>().len(), 16);
assert_eq!(initial_blocks, (1..=12).collect());
}
#[test]
fn bounded_metadata_fanout_order_uses_written_bucket_object_key() {
let bucket = "bounded-order-rotation-bucket";
let (object, bare_distribution, stored_distribution) = (0..1000)
.map(|index| format!("bounded-order-rotation-object-{index}"))
.find_map(|object| {
let bare_distribution = FileInfo::new(&object, 12, 4).erasure.distribution;
let stored_key = [bucket, object.as_str()].join("/");
let stored_distribution = FileInfo::new(&stored_key, 12, 4).erasure.distribution;
(bare_distribution != stored_distribution).then_some((object, bare_distribution, stored_distribution))
})
.expect("test should find a bucket/object pair with a different distribution rotation");
let order = bounded_metadata_fanout_order(bucket, &object, 16, 4);
let initial_stored_blocks: HashSet<_> = order
.iter()
.take(12)
.filter_map(|disk_index| stored_distribution.get(*disk_index).copied())
.collect();
let initial_bare_blocks: HashSet<_> = order
.iter()
.take(12)
.filter_map(|disk_index| bare_distribution.get(*disk_index).copied())
.collect();
assert_eq!(initial_stored_blocks, (1..=12).collect());
assert_ne!(
initial_bare_blocks,
(1..=12).collect(),
"test fixture must prove the bare object distribution would pick the wrong initial data-shard set"
);
}
#[tokio::test]
async fn bounded_metadata_early_stop_non_inline_fallback_schedules_beyond_initial_quorum() {
const DISKS: usize = 6;
let bucket = "bounded-data-get-six-disk-fanout-bucket";
let object = "bounded-data-get-six-disk-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 3)
.await
.expect("non-inline metadata should resolve after force-full-wait fallback");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"force-full-wait fallback must schedule disks beyond the initial write quorum"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_allows_verified_inline_data_get_quorum() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-fanout-bucket";
let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-object", 2, 3);
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |_| {}).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2);
let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let tracker = rename_fanout_barrier::observe_tasks(&object);
let calls = disk_call_counters::observe(&object);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
2,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("bounded fanout should hedge and pause the spare disk");
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read)
.await
.expect("verified inline quorum should return without the paused spare")
.expect("read task should join")
.expect("verified inline metadata should reach early-stop quorum");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"bounded fanout may schedule one spare before the verified inline quorum returns"
);
assert_eq!(
tracker.running(),
0,
"early-stop should drain spawned read_version tasks before returning"
);
assert_eq!(diagnostics.total_responses(), 3);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() {
const DISKS: usize = 6;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
let bucket = "bounded-inline-data-get-pending-shard-bucket";
let object =
object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS);
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo_with_geometry(
&disks,
bucket,
&object,
b"verified inline payload",
DATA_SHARDS,
PARITY_SHARDS,
|_| {},
)
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS);
let distribution_key = metadata_distribution_key(bucket, &object);
let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS)
.erasure
.distribution;
let paused_data_disk = *fanout_order
.iter()
.take(DATA_SHARDS)
.find(|disk_index| {
distribution
.get(**disk_index)
.is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index))
})
.expect("initial fanout should include a data shard to pause");
let hedged_parity_disk = fanout_order[DATA_SHARDS];
let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1];
let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let tracker = rename_fanout_barrier::observe_tasks(&object);
let calls = disk_call_counters::observe(&object);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
PARITY_SHARDS,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("initial data shard should pause before returning");
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("bounded fanout should hedge one parity disk while the data shard is pending");
assert!(
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
"inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("pending data shard should let the inline verifier finish");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
5,
"pending data-shard defer should not schedule the final parity disk"
);
assert_eq!(
calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk),
0,
"the remaining parity disk must stay unissued when pending data verification succeeds"
);
assert_eq!(
tracker.running(),
0,
"early-stop should drain spawned read_version tasks before returning"
);
assert_eq!(diagnostics.total_responses(), 5);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
let bucket = "legacy-inline-data-get-fanout-bucket";
let object = "legacy-inline-data-get-object";
let payload = b"legacy inline payload whose size is not divisible by the data shard count";
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, true).await;
let distribution = files
.first()
.map(|file| file.erasure.distribution.clone())
.expect("legacy fixture should include metadata");
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
let mut parts_metadata = vec![FileInfo::default(); 4];
for disk_index in order.into_iter().take(3) {
let block_index = distribution
.get(disk_index)
.copied()
.expect("legacy fixture distribution should cover every disk");
parts_metadata[disk_index] = files
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
.expect("legacy fixture should include every distributed shard")
.clone();
}
let candidate = parts_metadata
.iter()
.find(|file| file.name == object)
.expect("legacy fixture should include observed metadata")
.clone();
assert!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks)
.await
.is_none(),
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
);
}
#[tokio::test]
async fn data_read_early_stop_reports_inline_miss_reasons() {
let bucket = "inline-data-get-miss-reason-bucket";
let object = "inline-data-get-miss-reason-object";
let payload = b"verified inline payload";
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
let distribution = files
.first()
.map(|file| file.erasure.distribution.clone())
.expect("fixture should include metadata");
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
let mut parts_metadata = vec![FileInfo::default(); 4];
for disk_index in order.into_iter().take(3) {
let block_index = distribution
.get(disk_index)
.copied()
.expect("fixture distribution should cover every disk");
parts_metadata[disk_index] = files
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
.expect("fixture should include every distributed shard")
.clone();
}
let candidate = parts_metadata
.iter()
.find(|file| file.name == object)
.expect("fixture should include observed metadata")
.clone();
let data_disk = distribution
.iter()
.position(|block_index| *block_index == 1)
.expect("fixture distribution should include first data shard");
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
None
);
let mut not_inline = candidate.clone();
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, ¬_inline, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
);
let mut remote = candidate.clone();
remote.transition_status = TRANSITION_COMPLETE.to_string();
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE)
);
let mut transformed = candidate.clone();
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
);
let mut deleted = candidate.clone();
deleted.deleted = true;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
);
let mut zero_size = candidate.clone();
zero_size.size = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
);
let mut multipart = candidate.clone();
multipart.parts.push(multipart.parts[0].clone());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
);
let mut invalid_geometry = candidate.clone();
invalid_geometry.erasure.data_blocks = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
);
let mut missing_shard = parts_metadata.clone();
missing_shard[data_disk] = FileInfo::default();
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
);
let mut missing_payload = parts_metadata.clone();
missing_payload[data_disk].data = None;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
);
let mut identity_mismatch = parts_metadata.clone();
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
);
let mut corrupt = parts_metadata.clone();
if let Some(data) = corrupt[data_disk].data.as_mut() {
let mut corrupt_data = data.to_vec();
corrupt_data[0] ^= 0x01;
*data = Bytes::from(corrupt_data);
}
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
);
}
#[test]
#[serial_test::serial]
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
assert_metadata_fanout_lifecycle_records_real_early_stop_abort(
"lifecycle-inline-data-get-fanout-bucket",
"lifecycle-inline-data-get-object",
GET_OBJECT_PATH_LEGACY_DUPLEX,
None,
);
}
#[test]
#[serial_test::serial]
fn metadata_fanout_lifecycle_records_bounded_early_stop_abort() {
assert_metadata_fanout_lifecycle_records_real_early_stop_abort(
"lifecycle-bounded-inline-data-get-fanout-bucket",
"lifecycle-bounded-inline-data-get-object",
GET_OBJECT_PATH_LEGACY_DUPLEX,
Some("true"),
);
}
#[test]
#[serial_test::serial]
fn metadata_fanout_lifecycle_records_internal_meta_early_stop_abort_path() {
assert_metadata_fanout_lifecycle_records_real_early_stop_abort(
RUSTFS_META_BUCKET,
"buckets/.usage-cache/lifecycle-inline-data-get-object",
GET_OBJECT_PATH_INTERNAL_META,
None,
);
}
#[test]
#[serial_test::serial]
fn metadata_fanout_records_internal_meta_final_miss_path() {
const DISKS: usize = 4;
let bucket = RUSTFS_META_BUCKET;
let object = object_with_initial_data_shards(bucket, "buckets/.usage-cache/final-miss-inline-data-get-object", 2, 3);
let corrupt_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0];
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime should build");
let recorder = crate::test_metrics::CapturingRecorder::default();
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| {
if let Some(data) = files.get_mut(corrupt_shard - 1).and_then(|file| file.data.as_mut()) {
let mut corrupt = data.to_vec();
corrupt[0] ^= 0x01;
*data = Bytes::from(corrupt);
}
})
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let (parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed(
&disks, bucket, bucket, &object, "", true, false, false, true, 2,
)
.await
.expect("corrupt internal inline metadata should fall back to full fanout");
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
});
});
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
assert_eq!(
recorder.counter_value(
"rustfs_io_get_object_metadata_early_stop_total",
&[
("path", GET_OBJECT_PATH_INTERNAL_META),
("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
],
),
1,
"internal metadata final early-stop miss must retain its path label"
);
assert_eq!(
recorder.counter_value(
"rustfs_io_get_object_metadata_early_stop_total",
&[
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
],
),
0,
"internal metadata final early-stop miss must not leak into legacy_duplex"
);
assert_eq!(
recorder.histogram_values(
"rustfs_io_get_object_metadata_early_stop_saved_responses",
&[("path", GET_OBJECT_PATH_INTERNAL_META)]
),
vec![0.0],
"internal metadata final miss must record zero saved responses on internal_meta"
);
assert!(
recorder
.histogram_values(
"rustfs_io_get_object_metadata_early_stop_saved_responses",
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
)
.is_empty(),
"internal metadata final miss saved responses must not leak into legacy_duplex"
);
}
fn assert_metadata_fanout_lifecycle_records_real_early_stop_abort(
bucket: &'static str,
object_prefix: &str,
expected_path: &'static str,
bounded_fanout_env: Option<&'static str>,
) {
const DISKS: usize = 4;
let object = object_with_initial_data_shards(bucket, object_prefix, 2, 3);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime should build");
let recorder = crate::test_metrics::CapturingRecorder::default();
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |_| {}).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", bounded_fanout_env),
],
async {
let barrier_disk = if bounded_fanout_env.is_some() {
bounded_spare_disk_index(bucket, &object, 2, 2)
} else {
3
};
let barrier =
rename_fanout_barrier::arm(&object, barrier_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
2,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("metadata fanout should pause the spare metadata task");
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read)
.await
.expect("verified inline quorum should abort the paused eager fanout")
.expect("read task should join")
.expect("verified inline metadata should reach early-stop quorum");
assert_eq!(diagnostics.total_responses(), 3);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
});
});
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
assert_eq!(
recorder.histogram_values("rustfs_io_get_object_metadata_fanout_scheduled", &[("path", expected_path)]),
vec![4.0]
);
assert_eq!(
recorder.histogram_values("rustfs_io_get_object_metadata_fanout_completed", &[("path", expected_path)]),
vec![3.0]
);
assert_eq!(
recorder.histogram_values("rustfs_io_get_object_metadata_fanout_cancelled", &[("path", expected_path)]),
vec![1.0]
);
assert_eq!(
recorder.counter_value(
"rustfs_io_get_object_metadata_early_stop_total",
&[
("path", expected_path),
("decision", "hit"),
("reason", GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM),
],
),
1,
"early-stop hit must retain its path label"
);
let unexpected_path = if expected_path == GET_OBJECT_PATH_INTERNAL_META {
GET_OBJECT_PATH_LEGACY_DUPLEX
} else {
GET_OBJECT_PATH_INTERNAL_META
};
assert_eq!(
recorder.counter_value(
"rustfs_io_get_object_metadata_early_stop_total",
&[
("path", unexpected_path),
("decision", "hit"),
("reason", GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM),
],
),
0,
"early-stop hit must not leak into the other metadata path"
);
assert_eq!(
recorder.histogram_values("rustfs_io_get_object_metadata_early_stop_saved_responses", &[("path", expected_path)]),
vec![1.0]
);
assert!(
recorder
.histogram_values("rustfs_io_get_object_metadata_early_stop_saved_responses", &[("path", unexpected_path)])
.is_empty(),
"early-stop saved responses must not leak into the other metadata path"
);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_when_inline_data_is_corrupt() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-corrupt-bucket";
let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-corrupt-object", 2, 3);
let corrupt_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0];
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| {
if let Some(data) = files.get_mut(corrupt_shard - 1).and_then(|file| file.data.as_mut()) {
let mut corrupt = data.to_vec();
corrupt[0] ^= 0x01;
*data = Bytes::from(corrupt);
}
})
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(&object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, &object, "", true, false, false, true, 2)
.await
.expect("corrupt inline metadata should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"inline bitrot failure must keep metadata fanout open instead of aborting spares"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_when_inline_generation_differs() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-generation-bucket";
let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-generation-object", 2, 3);
let stale_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0];
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| {
if let Some(file) = files.get_mut(stale_shard - 1) {
file.version_id = Some(Uuid::new_v4());
}
})
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(&object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, &object, "", true, false, false, true, 2)
.await
.expect("mixed-generation inline metadata should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"inline data from a different metadata generation must not satisfy the data-read gate"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_when_inline_shard_identity_is_copied() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-copied-shard-bucket";
let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-copied-shard-object", 2, 3);
let data_shards = initial_data_shard_indexes(bucket, &object, 2, 3);
let source_shard = data_shards[0];
let target_shard = data_shards[1];
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| {
let copied = files[source_shard - 1].clone();
files[target_shard - 1].erasure.index = copied.erasure.index;
files[target_shard - 1].data = copied.data;
})
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2);
let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(&object);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
2,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("full-wait fallback should schedule the spare disk");
assert!(
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
"copied inline shard identity must not satisfy the early-stop data gate"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("copied inline shard identity should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"copied shard identity must keep metadata fanout open instead of aborting spares"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_for_transformed_inline_metadata() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-transform-bucket";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
async fn assert_full_wait(disks: &[Option<DiskStore>], bucket: &str, object: &str, mutate: impl FnOnce(&mut [FileInfo])) {
const DISKS: usize = 4;
install_inline_metadata_fanout_fileinfo(disks, bucket, object, b"verified inline payload", mutate).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(disks, bucket, bucket, object, "", true, false, false, true, 2)
.await
.expect("transformed inline metadata should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"transformed inline metadata must not pass the plaintext data-read gate"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
}
assert_full_wait(&disks, bucket, "bounded-inline-data-get-compressed-object", |files| {
for file in files {
rustfs_utils::http::insert_str(&mut file.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
}
})
.await;
assert_full_wait(&disks, bucket, "bounded-inline-data-get-encrypted-object", |files| {
for file in files {
file.metadata
.insert(rustfs_utils::http::headers::SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string());
}
})
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_for_unsafe_inline_metadata_shapes() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-unsafe-shape-bucket";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
async fn assert_full_wait(disks: &[Option<DiskStore>], bucket: &str, object: &str, mutate: impl FnOnce(&mut [FileInfo])) {
const DISKS: usize = 4;
install_inline_metadata_fanout_fileinfo(disks, bucket, object, b"verified inline payload", mutate).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(disks, bucket, bucket, object, "", true, false, false, true, 2)
.await
.expect("unsafe inline metadata shape should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"unsafe inline metadata shape {object} must not abort remaining metadata responses"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
}
assert_full_wait(&disks, bucket, "bounded-inline-data-get-remote-object", |files| {
for file in files {
file.transition_status = TRANSITION_COMPLETE.to_string();
}
})
.await;
assert_full_wait(&disks, bucket, "bounded-inline-data-get-zero-size-object", |files| {
for file in files {
file.size = 0;
if let Some(part) = file.parts.first_mut() {
part.size = 0;
}
}
})
.await;
assert_full_wait(&disks, bucket, "bounded-inline-data-get-multipart-object", |files| {
for file in files {
let mut second_part = file.parts[0].clone();
second_part.number = 2;
file.parts.push(second_part);
}
})
.await;
assert_full_wait(&disks, bucket, "bounded-inline-data-get-part-size-mismatch-object", |files| {
for file in files {
if let Some(part) = file.parts.first_mut() {
part.size = part.size.saturating_add(1);
}
}
})
.await;
assert_full_wait(&disks, bucket, "bounded-inline-data-get-oversize-object", |files| {
for file in files {
let oversize = file.erasure.block_size.saturating_add(1);
file.size = i64::try_from(oversize).expect("test block size should fit i64");
if let Some(part) = file.parts.first_mut() {
part.size = oversize;
}
}
})
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_full_waits_for_purge_pending_inline_payload() {
const DISKS: usize = 4;
let bucket = "bounded-inline-data-get-purge-pending-bucket";
let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-purge-pending-object", 2, 3);
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| {
for file in files {
rustfs_utils::http::insert_str(
&mut file.metadata,
rustfs_utils::http::SUFFIX_PURGESTATUS,
"target=PENDING;".to_string(),
);
let replication_state = crate::bucket::replication::ReplicationState {
version_purge_status_internal: Some("target=PENDING;".to_string()),
purge_targets: crate::bucket::replication::version_purge_statuses_map("target=PENDING;"),
..Default::default()
};
file.replication_state_internal =
Some(crate::bucket::replication::replication_state_to_filemeta(&replication_state));
file.deleted = true;
assert!(
!file.is_canonical_delete_marker(),
"test fixture must remain an erasure-backed purge-pending payload"
);
}
})
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2);
let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(&object);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
2,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("purge-pending fallback should schedule the spare disk");
assert!(
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
"purge-pending payload metadata must not satisfy the inline data-read gate"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("purge-pending inline payload should fall back to full fanout");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"purge-pending payload must keep metadata fanout open instead of aborting spares"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() {
const DISKS: usize = 4;
let bucket = "bounded-data-get-hedge-bucket";
let object = "bounded-data-get-hedge-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(object);
let disks_for_read = disks.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("third scheduled read_version should pause at the deterministic barrier");
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk");
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
assert!(
pending.is_err(),
"non-inline data reads must not return before the paused metadata response"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("healthy spare metadata should resolve");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() {
const DISKS: usize = 4;
let bucket = "bounded-data-get-default-bucket";
let object = "bounded-data-get-default-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
],
async {
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(object);
let disks_for_read = disks.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("default bounded non-inline read should schedule the paused metadata task");
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect(
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
);
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
assert!(
pending.is_err(),
"default non-inline data reads must not return before the paused metadata response"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("default data-read metadata should resolve");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn bounded_metadata_early_stop_falls_back_to_full_fanout_on_data_read_error() {
const DISKS: usize = 4;
let bucket = "bounded-data-get-error-bucket";
let object = "bounded-data-get-error-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, Some(0)).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let calls = disk_call_counters::observe(object);
let (_, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
.await
.expect("metadata fanout should complete after falling back to all disks");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"a data-read error must force bounded fanout to schedule every disk before returning"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert!(
errs.iter()
.any(|err| err.as_ref().is_some_and(|err| matches!(err, DiskError::FileNotFound)))
);
},
)
.await;
drop(dirs);
}
const BARRIER_PAUSE_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
#[tokio::test]
async fn rename_tail_join_error_fails_safe_to_heal() {
let submissions = Arc::new(std::sync::Mutex::new(Vec::new()));
let captured = Arc::clone(&submissions);
let cleanup_called = Arc::new(std::sync::Mutex::new(false));
let cleanup_captured = Arc::clone(&cleanup_called);
let finalize_called = Arc::new(std::sync::Mutex::new(false));
let finalize_captured = Arc::clone(&finalize_called);
let tail_drain = tokio::spawn(async {
panic!("injected rename tail join failure");
#[allow(unreachable_code)]
None
});
let (release, released) = tokio::sync::oneshot::channel::<bool>();
drop(release);
finish_rename_tail_heal(
tail_drain,
released,
(),
rustfs_heal_contracts::heal_channel::HealChannelRequest::default(),
move || async move {
*finalize_captured.lock().expect("finalize recorder should not poison") = true;
},
move |(), _| async move {
*cleanup_captured.lock().expect("cleanup recorder should not poison") = true;
},
|request| async move {
captured.lock().expect("submission recorder should not poison").push(request);
},
)
.await;
assert_eq!(
submissions.lock().expect("submission recorder should not poison").len(),
1,
"a tail JoinError must fail safe to heal"
);
assert!(
!*cleanup_called.lock().expect("cleanup recorder should not poison"),
"a tail JoinError must preserve staging that an in-flight disk task may still need"
);
assert!(
*finalize_called.lock().expect("finalize recorder should not poison"),
"a completed tail JoinError must release resources needed only by disk rename tasks"
);
}
fn rename_barrier_fileinfos(object: &str, count: usize) -> Vec<FileInfo> {
(0..count).map(|_| metadata_test_fileinfo(object)).collect()
}
fn rename_commit_fileinfos(object: &str, count: usize, etag: &str) -> Vec<FileInfo> {
(0..count)
.map(|idx| {
let mut file_info = metadata_test_fileinfo(object);
file_info.mod_time = Some(OffsetDateTime::now_utc());
file_info.erasure.index = idx + 1;
file_info.data = Some(Bytes::from_static(b"inline-body"));
file_info.set_inline_data();
file_info.metadata.insert("etag".to_string(), etag.to_string());
file_info
})
.collect()
}
#[tokio::test]
async fn rename_data_skips_offline_placeholder_when_validating_new_metadata() {
let (_dirs, mut online_disks) = call_counter_local_disks("rename-validation-bucket", 1).await;
let online_disk = online_disks.pop().expect("one test disk should be present");
let mut file_info = metadata_test_fileinfo("rename-unassigned-index");
file_info.erasure.index = 0;
let err = SetDisks::rename_data(
&[None, online_disk],
RUSTFS_META_TMP_BUCKET,
"source",
&[FileInfo::default(), file_info],
"bucket",
"object",
1,
)
.await
.expect_err("the missing staged source must fail after metadata validation");
assert_ne!(err, DiskError::FileCorrupt);
}
#[tokio::test]
async fn rename_data_accepts_canonical_delete_marker() {
let bucket = "rename-delete-marker-bucket";
let object = "object";
let (_dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
let online_disk = online_disks.pop().expect("one test disk slot should be present");
let disk = online_disk.as_ref().expect("test disk should be online");
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
}
let version_id = Uuid::new_v4();
let mut marker = metadata_test_delete_marker(object, version_id, OffsetDateTime::now_utc());
marker
.metadata
.insert("x-rustfs-internal-purgestatus".to_string(), "pending".to_string());
SetDisks::rename_data(
&[None, online_disk.clone()],
RUSTFS_META_TMP_BUCKET,
"source",
&[FileInfo::default(), marker],
bucket,
object,
1,
)
.await
.expect("canonical delete marker should commit without erasure payload");
let stored = disk
.read_version("", bucket, object, &version_id.to_string(), &ReadOptions::default())
.await
.expect("committed delete marker should be readable");
assert!(stored.deleted);
assert_eq!(stored.version_id, Some(version_id));
assert_eq!(stored.erasure.index, 0);
assert_eq!(stored.metadata.get("x-rustfs-internal-purgestatus").map(String::as_str), Some("pending"));
}
#[tokio::test]
async fn rename_data_preserves_null_delete_marker_type() {
let bucket = "rename-null-marker-bucket";
let object = "object";
let (_dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
let online_disk = online_disks.pop().expect("one test disk slot should be present");
let disk = online_disk.as_ref().expect("test disk should be online");
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
}
let marker = metadata_test_delete_marker(object, Uuid::new_v4(), OffsetDateTime::now_utc());
let marker = FileInfo {
version_id: None,
..marker
};
SetDisks::rename_data(
std::slice::from_ref(&online_disk),
RUSTFS_META_TMP_BUCKET,
"source",
&[marker],
bucket,
object,
1,
)
.await
.expect("null delete marker should commit");
let stored = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("null delete marker should remain readable");
assert!(stored.deleted);
assert_eq!(stored.version_id, None);
assert!(stored.is_canonical_delete_marker());
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_reclaims_synthetic_inline_rollback_dir_after_commit() {
let bucket = "rename-inline-rollback-bucket";
let object = "object";
let (dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
let online_disk = online_disks.pop().expect("one test disk slot should be present");
let disk = online_disk.as_ref().expect("test disk should be online");
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
}
let disk_root = dirs[0].path();
let mut inline_fi = metadata_test_fileinfo(object);
inline_fi.data = Some(Bytes::from_static(b"inline-body"));
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
std::fs::create_dir_all(disk_root.join(RUSTFS_META_TMP_BUCKET).join("tmp-inline"))
.expect("inline staging dir should be created");
let commit = SetDisks::rename_data_owned(
std::slice::from_ref(&online_disk),
(RUSTFS_META_TMP_BUCKET, "tmp-inline"),
vec![inline_fi],
(bucket, object),
1,
true,
)
.await
.expect("inline version should commit");
if let Some(td) = commit.tail_drain {
td.await
.expect("inline commit tail drain must succeed")
.expect("committed inline tail must report convergence");
}
let new_data_dir = Uuid::new_v4();
let mut streaming_fi = metadata_test_fileinfo(object);
streaming_fi.data_dir = Some(new_data_dir);
streaming_fi.mod_time = Some(OffsetDateTime::now_utc());
let staged_data_dir = disk_root
.join(RUSTFS_META_TMP_BUCKET)
.join("tmp-streaming")
.join(new_data_dir.to_string());
std::fs::create_dir_all(&staged_data_dir).expect("streaming staging dir should be created");
std::fs::write(staged_data_dir.join("part.1"), b"streamed-body").expect("staged part should be written");
let commit = SetDisks::rename_data_owned(
std::slice::from_ref(&online_disk),
(RUSTFS_META_TMP_BUCKET, "tmp-streaming"),
vec![streaming_fi],
(bucket, object),
1,
true,
)
.await
.expect("non-inline overwrite should commit");
if let Some(td) = commit.tail_drain {
td.await
.expect("non-inline overwrite tail drain must succeed")
.expect("committed non-inline tail must report convergence");
}
let mut leftovers: Vec<String> = std::fs::read_dir(disk_root.join(bucket).join(object))
.expect("committed object dir should be readable")
.map(|entry| {
entry
.expect("object dir entry should be readable")
.file_name()
.to_string_lossy()
.into_owned()
})
.collect();
leftovers.sort();
assert_eq!(
leftovers,
vec![new_data_dir.to_string(), STORAGE_FORMAT_FILE.to_string()],
"only the committed data dir and xl.meta may remain — synthetic rollback residue breaks DeleteBucket"
);
}
#[tokio::test]
async fn rename_delete_marker_quorum_failure_restores_existing_metadata() {
let bucket = "rename-marker-quorum-bucket";
let object = "object";
let (_dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
let online_disk = online_disks.pop().expect("one test disk slot should be present");
let disk = online_disk.as_ref().expect("test disk should be online");
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
}
let old_version_id = Uuid::new_v4();
let mut old = metadata_test_fileinfo(object);
old.version_id = Some(old_version_id);
old.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, old)
.await
.expect("old metadata should be written");
let marker_version_id = Uuid::new_v4();
let marker = metadata_test_delete_marker(object, marker_version_id, OffsetDateTime::now_utc());
let err = SetDisks::rename_data(
&[online_disk.clone(), None],
RUSTFS_META_TMP_BUCKET,
"source",
&[marker, FileInfo::default()],
bucket,
object,
2,
)
.await
.expect_err("quorum-minus-one marker commit should fail");
assert_eq!(err, DiskError::ErasureWriteQuorum);
let restored = disk
.read_version("", bucket, object, &old_version_id.to_string(), &ReadOptions::default())
.await
.expect("old metadata should remain after rollback");
assert!(!restored.deleted);
assert_eq!(restored.version_id, Some(old_version_id));
assert!(matches!(
disk.read_version("", bucket, object, &marker_version_id.to_string(), &ReadOptions::default())
.await,
Err(DiskError::FileVersionNotFound)
));
}
#[tokio::test]
async fn rename_fanout_barrier_pauses_and_reports_running_background_tasks() {
const DISKS: usize = 4;
let bucket = "rename-barrier-bucket";
let object = "rename-barrier-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
let file_infos = rename_barrier_fileinfos(object, DISKS);
let tracker = rename_fanout_barrier::observe_tasks(object);
assert_eq!(tracker.running(), 0, "no fan-out tasks before rename starts");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename_fut = SetDisks::rename_data(&disks, bucket, object, &file_infos, bucket, object, DISKS - 1);
let control_fut = async {
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("fan-out must reach the armed rename barrier");
assert!(barrier.is_paused(), "target task must be parked at the checkpoint");
assert!(tracker.running() >= 1, "a background rename task must still be in flight while paused");
barrier.release();
};
let (_rename_res, ()) = tokio::join!(rename_fut, control_fut);
assert_eq!(tracker.running(), 0, "no background rename task may remain once the fan-out has drained");
drop(dirs);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_fanout_drains_after_caller_cancellation() {
const DISKS: usize = 4;
let bucket = "rename-cancel-bucket";
let object = "rename-cancel-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
let marker = metadata_test_delete_marker(object, Uuid::new_v4(), OffsetDateTime::now_utc());
let file_infos = vec![marker; DISKS];
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename =
tokio::spawn(
async move { SetDisks::rename_data(&disks, bucket, object, &file_infos, bucket, object, DISKS - 1).await },
);
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("rename fan-out must reach the armed barrier");
rename.abort();
assert!(
rename
.await
.expect_err("aborted caller should report cancellation")
.is_cancelled(),
"caller task should be cancelled, not panic"
);
assert!(tracker.running() >= 1, "the coordinator must retain in-flight disk mutations");
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while tracker.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled caller's disk mutations must drain");
for (idx, dir) in dirs.iter().enumerate() {
assert!(
dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
"disk {idx} must finish the rename after caller cancellation"
);
}
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_without_tail_handoff_waits_after_write_quorum() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-tail-success-bucket";
let object = "rename-tail-success-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "tail-success-etag");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename = SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3);
tokio::pin!(rename);
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
result = &mut rename => panic!("rename_data returned before the armed fan-out barrier: {result:?}"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut rename).await.is_err(),
"serial rename_data waits for the paused fan-out disk even after write quorum"
);
barrier.release();
rename
.await
.expect("tail success must complete the rename after the barrier is released");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must contain the tail-success commit after reopen: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("tail-success-etag"),
"disk {idx} must expose the same committed metadata after tail success and reopen"
);
}
drop(dirs);
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_returns_after_write_quorum_and_drains_tail_success() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-tail-success-bucket";
let object = "rename-early-ack-tail-success-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "early-tail-success-etag");
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let mut rename = Box::pin(SetDisks::rename_data_owned(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
3,
true,
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename_data returned before the armed fan-out barrier"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
let commit = rename
.await
.expect("early ACK must return once the other three disks satisfy write quorum");
assert!(
tracker.running() >= 1,
"the paused tail disk must continue in the background after early ACK"
);
assert_eq!(
commit.convergence,
RenameConvergence::PartialCommit,
"the quorum snapshot sees the unresolved tail as missing"
);
assert!(
!commit.needs_immediate_heal(),
"a pending tail must defer heal admission until its final observations"
);
let tail_drain = commit.tail_drain.expect("early ACK must return a tail drain");
barrier.release();
let tail_outcome = tokio::time::timeout(BARRIER_PAUSE_GUARD, tail_drain)
.await
.expect("background tail disk must drain after release")
.expect("background tail task must join")
.expect("a committed early-ACK tail must return final convergence");
let final_convergence = tail_outcome.convergence;
assert_eq!(
final_convergence,
RenameConvergence::AllSuccessIdentical,
"a successful tail must clear the quorum snapshot's false partial commit"
);
assert!(!final_convergence.needs_heal(), "a successful tail must not enqueue heal after release");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must contain the early-ACK commit after reopen: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("early-tail-success-etag"),
"disk {idx} must expose the same committed metadata after background tail success and reopen"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_tail_recovers_final_old_data_dir_quorum() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-old-dir-quorum-bucket";
let object = "rename-early-ack-old-dir-quorum-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let minority_old_dir = Uuid::new_v4();
let majority_old_dir = Uuid::new_v4();
let old_dirs = [minority_old_dir, majority_old_dir, majority_old_dir, majority_old_dir];
let old_mod_time = OffsetDateTime::now_utc();
for (idx, disk) in disks.iter().flatten().enumerate() {
let mut old = metadata_test_fileinfo(object);
old.data_dir = Some(old_dirs[idx]);
old.mod_time = Some(old_mod_time);
disk.write_all(bucket, &format!("{object}/{}/part.1", old_dirs[idx]), Bytes::from_static(b"old-body"))
.await
.expect("old data shard should be written");
disk.write_metadata(bucket, bucket, object, old)
.await
.expect("old metadata should be written");
}
let file_infos = rename_commit_fileinfos(object, DISKS, "early-old-dir-quorum-etag");
let barrier = rename_fanout_barrier::arm(object, 3, rename_fanout_barrier::PHASE_RENAME);
let mut rename = Box::pin(SetDisks::rename_data_owned(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
3,
true,
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename_data returned before the armed fan-out barrier"),
}
})
.await
.expect("the final majority-vote disk must pause before rename");
let commit = rename.await.expect("the first three disks must satisfy write quorum");
assert_eq!(commit.data_dir, None, "two matching old dirs are not a write quorum");
assert!(
commit.cleanup_disks.iter().all(Option::is_none),
"the quorum snapshot must not schedule cleanup without an old-dir quorum"
);
let tail_drain = commit.tail_drain.expect("early ACK must return a tail drain");
barrier.release();
let outcome = tokio::time::timeout(BARRIER_PAUSE_GUARD, tail_drain)
.await
.expect("the paused majority-vote disk must drain after release")
.expect("background tail task must join")
.expect("a committed early-ACK tail must return final observations");
let cleanup_indexes: Vec<_> = outcome.cleanup.iter().map(|target| target.disk_index).collect();
assert_eq!(
cleanup_indexes,
vec![1, 2, 3],
"the final old-dir quorum must recover both snapshot and tail cleanup targets"
);
let set = io_primitives_test_set(disks, 2).await;
set.cleanup_rename_tail(outcome.cleanup, bucket, object, None, None).await;
assert!(
dirs[0]
.path()
.join(bucket)
.join(object)
.join(minority_old_dir.to_string())
.exists(),
"the minority old data dir must not be reclaimed"
);
for dir in &dirs[1..] {
assert!(
!dir.path()
.join(bucket)
.join(object)
.join(majority_old_dir.to_string())
.exists(),
"every disk in the final old-dir quorum must be reclaimed"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_tail_failure_does_not_expose_partial_fresh_after_reopen() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-tail-failure-bucket";
let object = "rename-early-ack-tail-failure-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "early-tail-failure-etag");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
let mut rename = Box::pin(SetDisks::rename_data_owned(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
3,
true,
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename_data returned before the armed fan-out barrier"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
let commit = rename
.await
.expect("early ACK must return after write quorum before the tail failure");
assert!(
!commit.needs_immediate_heal(),
"a pending failed tail must still defer heal until the failure is observed"
);
let tail_drain = commit.tail_drain.expect("early ACK must return a tail drain");
barrier.release();
let tail_outcome = tokio::time::timeout(BARRIER_PAUSE_GUARD, tail_drain)
.await
.expect("background tail failure must drain after release")
.expect("background tail task must join")
.expect("a committed early-ACK tail must return final convergence");
let final_convergence = tail_outcome.convergence;
assert_eq!(
final_convergence,
RenameConvergence::PartialCommit,
"the observed tail failure must require heal"
);
assert!(final_convergence.needs_heal(), "a real tail failure must enqueue heal after drain");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let read = reopened.read_version("", bucket, object, "", &ReadOptions::default()).await;
if idx == 0 {
assert!(
matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"failed background tail disk must not expose a partial fresh commit after reopen: {read:?}"
);
} else {
let stored = read.unwrap_or_else(|err| {
panic!("quorum disk {idx} must contain the early-ACK commit after reopen: {err:?}")
});
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("early-tail-failure-etag"),
"quorum disk {idx} must expose the committed metadata after reopen"
);
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_tail_failure_preserves_overwrite_after_reopen() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-overwrite-tail-failure-bucket";
let object = "rename-early-ack-overwrite-tail-failure-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.metadata.insert("etag".to_string(), "early-old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be written before early overwrite");
}
let file_infos = rename_commit_fileinfos(object, DISKS, "early-new-etag");
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
let mut rename = Box::pin(SetDisks::rename_data_owned(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
3,
true,
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename_data returned before the armed fan-out barrier"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
rename
.await
.expect("early ACK must return after write quorum before overwrite tail failure");
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while tracker.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("background overwrite tail failure must drain after release");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must have a readable version after early overwrite: {err:?}"));
let expected_etag = if idx == 0 { "early-old-etag" } else { "early-new-etag" };
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some(expected_etag),
"disk {idx} must keep the correct early-ACK overwrite visibility after reopen"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_background_drains_after_caller_cancellation() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-cancel-bucket";
let object = "rename-early-ack-cancel-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "early-cancel-etag");
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let mut rename = Box::pin(SetDisks::rename_data_owned(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
3,
true,
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename_data returned before the armed fan-out barrier"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
drop(rename);
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while tracker.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("background rename fan-out must drain after caller cancellation");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must contain the cancelled early-ACK commit after drain: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("early-cancel-etag"),
"disk {idx} must expose the background-drained commit after caller cancellation"
);
}
})
.await;
}
#[tokio::test]
async fn rename_rollback_incomplete_inspection_rejection_is_not_recovery() {
fn reject_inspection(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
assert_eq!(request.bucket, "rollback-inspection");
assert_eq!(request.object_prefix.as_deref(), Some("object"));
assert_eq!(request.dry_run, Some(true), "failed minority must never become a mutating heal source");
assert_eq!(request.remove_corrupted, Some(false));
assert_eq!(request.recreate_missing, Some(false));
assert_eq!(request.recursive, Some(false));
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full) })
}
let result = inspect_incomplete_rename_rollback(&[], "rollback-inspection", "object", reject_inspection).await;
assert!(matches!(result, ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full)));
}
#[tokio::test]
async fn rename_rollback_incomplete_cancelled_task_is_not_success() {
let task = tokio::spawn(std::future::pending::<disk::error::Result<()>>());
task.abort();
assert_eq!(rename_rollback_task_outcome(task.await), RenameRollbackOutcome::Cancelled);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
const WRITE_QUORUM: usize = 3;
for overwrite in [false, true] {
for success_count in [0, WRITE_QUORUM - 1, WRITE_QUORUM] {
for fault in [
None,
Some(rollback_fault_injection::Fault::Io),
Some(rollback_fault_injection::Fault::Panic),
] {
let mut previous = None;
for early_ack in [false, true] {
let bucket = "rename-rollback-matrix";
let object = format!("object-{overwrite}-{success_count}-{fault:?}-{early_ack}");
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
if overwrite {
let mut old = metadata_test_fileinfo(&object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, &object, old.clone())
.await
.expect("old version must be staged");
}
}
let _rename_fault =
rename_fault_injection::fail_rename_on(&object, &(success_count..DISKS).collect::<Vec<_>>());
let _undo_fault = fault.map(|fault| rollback_fault_injection::arm(&object, 0, fault));
let receipt = RenameRollbackReceipt::default();
let result = SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(&object, DISKS, "new-etag"),
(bucket, &object),
early_ack,
RenameDataFenceOptions::new(WRITE_QUORUM, None).with_rollback_receipt(receipt.clone()),
)
.await;
let actual_error = match result {
Ok(commit) => {
assert_eq!(success_count, WRITE_QUORUM);
if let Some(tail) = commit.tail_drain {
tail.await
.expect("committed tail should join")
.expect("committed tail should converge");
}
assert!(
receipt.0.get().is_none(),
"a quorum commit must not enter rollback even when undo faults are armed"
);
None
}
Err(err) => {
assert!(success_count < WRITE_QUORUM);
let report = receipt
.0
.get()
.expect("both failure paths must publish per-disk rollback evidence");
assert_eq!(report.disks.len(), DISKS);
for (idx, outcome) in report.disks.iter().enumerate() {
assert_eq!(outcome.disk_index, idx);
let expected = if idx >= success_count {
RenameRollbackOutcome::NotAttempted(DiskError::other(
"injected rename failure (test-only)",
))
} else if idx == 0 {
match fault {
Some(rollback_fault_injection::Fault::Io) => {
RenameRollbackOutcome::Failed(DiskError::FaultyDisk)
}
Some(rollback_fault_injection::Fault::Panic) => RenameRollbackOutcome::Panicked,
None => RenameRollbackOutcome::Succeeded,
Some(_) => unreachable!("matrix only injects undo faults"),
}
} else {
RenameRollbackOutcome::Succeeded
};
assert_eq!(outcome.outcome, expected);
if overwrite && idx < success_count {
let backup = dirs[idx]
.path()
.join(bucket)
.join(&object)
.join(outcome.rollback_dir.expect("overwrite needs rollback dir").to_string())
.join(STORAGE_FORMAT_FILE_BACKUP);
assert_eq!(
backup.exists(),
outcome.outcome.needs_recovery(),
"failed undo must retain its only old-version backup"
);
}
}
assert_eq!(receipt.is_incomplete(), success_count > 0 && fault.is_some());
Some(err)
}
};
if let Some(expected_error) = previous.as_ref() {
assert_eq!(
&actual_error, expected_error,
"early ACK must preserve the original full-wait quorum error"
);
}
previous = Some(actual_error);
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let read = reopened
.read_version(
"",
bucket,
&object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await;
let keeps_new =
idx < success_count && (success_count == WRITE_QUORUM || (idx == 0 && fault.is_some()));
if keeps_new || overwrite {
let stored = read.expect("old or committed version must survive reopen");
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some(if keeps_new { "new-etag" } else { "old-etag" })
);
assert_eq!(
stored.data.as_deref(),
Some(if keeps_new {
b"inline-body".as_slice()
} else {
b"old-inline-body".as_slice()
}),
"object={object}, disk={idx}, keeps_new={keeps_new}"
);
} else {
assert!(
matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"fresh rollback must not expose data: {read:?}"
);
}
}
}
}
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for fault in [
rollback_fault_injection::Fault::IoAfterRename,
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
rollback_fault_injection::Fault::PanicAfterRename,
] {
let bucket = "rename-tail-unknown";
let object = format!("tail-{fault:?}");
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let receipt = RenameRollbackReceipt::default();
let _fault = rollback_fault_injection::arm(&object, 0, fault);
let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier_phase::RENAME);
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(&object, 4, "new-etag"),
(bucket, &object),
true,
RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()),
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("tail barrier must precede quorum ACK"),
}
})
.await
.expect("tail reaches the barrier");
let commit = tokio::time::timeout(BARRIER_PAUSE_GUARD, rename)
.await
.expect("quorum must ACK before tail release")
.expect("three disks commit");
barrier.release();
let tail = commit
.tail_drain
.expect("early ACK owns a tail")
.await
.expect("tail coordinator joins")
.expect("committed tail reports convergence");
assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
assert!(receipt.0.get().is_none(), "post-ACK errors must never start rollback");
for dir in &dirs {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version(
"",
bucket,
&object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("all actual writes survive despite a lost tail acknowledgement");
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for fault in [
rollback_fault_injection::Fault::Io,
rollback_fault_injection::Fault::IoAfterRename,
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
rollback_fault_injection::Fault::PanicAfterRename,
rollback_fault_injection::Fault::CoordinatorPanic,
] {
for early_ack in [false, true] {
let bucket = "rollback-data-dirs";
let object = format!("object-{early_ack}-{fault:?}");
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let old_data_dir = Uuid::new_v4();
let new_data_dir = Uuid::new_v4();
let mut old = metadata_test_fileinfo(&object);
old.data_dir = Some(old_data_dir);
old.mod_time = Some(OffsetDateTime::now_utc());
let mut infos = Vec::new();
for (idx, disk) in disks.iter().enumerate() {
let disk = disk.as_ref().expect("fixture disk should be present");
disk.write_metadata(bucket, bucket, &object, old.clone())
.await
.expect("old metadata should be staged");
let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string());
tokio::fs::create_dir_all(&old_dir)
.await
.expect("old data directory should exist");
tokio::fs::write(old_dir.join("part.1"), b"old-data")
.await
.expect("old shard should exist");
let source = dirs[idx]
.path()
.join(RUSTFS_META_TMP_BUCKET)
.join("source")
.join(new_data_dir.to_string());
tokio::fs::create_dir_all(&source)
.await
.expect("new data directory should be staged");
tokio::fs::write(source.join("part.1"), b"new-data")
.await
.expect("new shard should be staged");
let mut fi = metadata_test_fileinfo(&object);
fi.data_dir = Some(new_data_dir);
fi.erasure.index = idx + 1;
fi.mod_time = Some(OffsetDateTime::now_utc());
infos.push(fi);
}
let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(&object, 0, fault);
let receipt = RenameRollbackReceipt::default();
assert!(
SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
infos,
(bucket, &object),
early_ack,
RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()),
)
.await
.is_err()
);
assert!(receipt.is_incomplete());
if !matches!(fault, rollback_fault_injection::Fault::Io) {
assert!(
matches!(
receipt.0.get().expect("indeterminate report").disks[0].outcome,
RenameRollbackOutcome::Indeterminate(_)
),
"post-mutation failure must not be classified as unattempted"
);
}
let coordinator_failed = matches!(fault, rollback_fault_injection::Fault::CoordinatorPanic);
for (idx, dir) in dirs.iter().enumerate() {
let root = dir.path().join(bucket).join(&object);
assert_eq!(
tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1"))
.await
.expect("old data must survive failed overwrite"),
b"old-data"
);
let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
assert_eq!(
backup.exists(),
idx == 0 || (coordinator_failed && idx == 1),
"unknown mutations must retain the old-version backup"
);
if idx >= 2 {
let staged = dir
.path()
.join(RUSTFS_META_TMP_BUCKET)
.join("source")
.join(new_data_dir.to_string())
.join("part.1");
assert_eq!(
tokio::fs::read(staged)
.await
.expect("failed-write staging must remain available"),
b"new-data"
);
}
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("metadata should survive reopen");
assert_eq!(
stored.data_dir,
Some(if idx == 0 || (coordinator_failed && idx == 1) {
new_data_dir
} else {
old_data_dir
})
);
}
}
}
})
.await;
}
#[cfg(not(windows))]
async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) {
use crate::disk::os;
use futures::FutureExt;
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
],
async {
const DISKS: usize = 4;
let bucket = "namespace-physical-tail";
let object = "inline-overwrite";
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.size = 15;
old.parts.clear();
old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None);
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag");
let mut hooks = Vec::new();
let mut entered = Vec::new();
let mut releases = Vec::new();
let mut publication_paths = Vec::new();
for (disk, info) in disks.iter().flatten().zip(&mut infos) {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("the old inline version must be readable before overwrite");
info.size = 11;
info.parts.clear();
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
let crate::disk::Disk::Local(local) = disk.as_ref() else {
panic!("physical publication fixture requires local disks");
};
let destination = local
.get_disk()
.get_object_path_for_io(bucket, object)
.expect("the publication path must resolve through the disk's mount lease");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
hooks.push(os::prepared_publication_test_hooks::install(
&destination.join(STORAGE_FORMAT_FILE),
move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
},
));
entered.push(entered_rx);
releases.push(release_tx);
publication_paths.push(destination);
}
let namespace_owner = ctx.begin_namespace_commit();
let namespace_probe = Arc::downgrade(&namespace_owner);
let receipt = RenameRollbackReceipt::default();
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
infos,
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(namespace_owner)),
));
tokio::time::timeout(Duration::from_secs(10), async {
tokio::select! {
signals = join_all(entered) => {
assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter");
}
_ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"),
}
})
.await
.expect("all four prepared metadata renames must reach their blocking syscall");
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), 1);
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
let result = tokio::time::timeout(Duration::from_secs(5), rename)
.await
.expect("ordinary disk timeout must not wait for the physical rename");
assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum");
let report = receipt.0.get().expect("failed fanout must finish rollback accounting");
assert_eq!(report.disks.len(), DISKS);
assert!(
report
.disks
.iter()
.all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout)))
);
let pending_before_release = ctx.namespace_commits_pending();
let owner_alive_before_release = namespace_probe.upgrade().is_some();
let old_snapshot_generation = ctx.namespace_commit_generation();
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
let root = disk.path();
assert!(
os::acquire_rename_data_mutation_lease(&root, bucket, destination)
.now_or_never()
.is_none(),
"the physical publication must still own object serialization after the async timeout"
);
assert!(
root.join(RUSTFS_META_TMP_BUCKET)
.join("source")
.join(STORAGE_FORMAT_FILE)
.exists()
);
let stored = disk
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("a scanner can still read the complete old metadata while publication is paused");
assert_eq!(stored.size, 15);
assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation);
drop(releases);
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
let root = disk.path();
let lease = tokio::time::timeout(
Duration::from_secs(5),
os::acquire_rename_data_mutation_lease(&root, bucket, destination),
)
.await
.expect("released physical publishers must drain");
drop(lease);
}
for dir in &dirs {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("the detached prepared rename must actually publish after timeout");
assert_eq!(stored.size, 11);
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
tokio::time::timeout(Duration::from_secs(5), async {
while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("released physical publishers must release namespace ownership");
let generation_after_publication = ctx.namespace_commit_generation();
assert!(!ctx.namespace_commits_pending());
assert!(namespace_probe.upgrade().is_none());
assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence");
assert!(
pending_before_release && owner_alive_before_release,
"physical publication outlived namespace accounting: early_ack={allow_early_ack}, \
pending={pending_before_release}, owner_alive={owner_alive_before_release}, \
old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}"
);
assert!(
generation_after_publication > old_snapshot_generation,
"physical completion must invalidate the scanner's old metadata snapshot"
);
},
)
.await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() {
assert_namespace_owner_survives_physical_publication_timeout(false).await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() {
assert_namespace_owner_survives_physical_publication_timeout(true).await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() {
use crate::disk::os;
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
],
async {
let bucket = "physical-ack-tail";
let object = "ack-object";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut infos = rename_commit_fileinfos(object, 4, "new-etag");
for info in &mut infos {
info.size = 11;
info.parts.clear();
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
}
let disk = disks[3].as_ref().expect("tail disk");
let crate::disk::Disk::Local(local) = disk.as_ref() else {
panic!("local fixture");
};
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let owner = ctx.begin_namespace_commit();
let owner_probe = Arc::downgrade(&owner);
let receipt = RenameRollbackReceipt::default();
let caller_receipt = receipt.clone();
let caller_disks = disks.clone();
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let commit = SetDisks::rename_data_owned_with_fence(
&caller_disks,
(RUSTFS_META_TMP_BUCKET, "source"),
infos,
(bucket, object),
true,
RenameDataFenceOptions::new(3, None)
.with_namespace_commit_guard(Some(owner))
.with_rollback_receipt(caller_receipt),
)
.await
.expect("three real disk publications must produce a successful ACK");
assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK");
std::future::pending::<()>().await;
});
let mut commit = tokio::time::timeout(Duration::from_secs(10), async {
entered_rx.await.expect("physical tail entry");
ack_rx
.await
.expect("ACK must arrive while the fourth disk is physically paused")
})
.await
.expect("successful quorum ACK must not wait for its physical tail");
assert_eq!(commit.online_disks.iter().flatten().count(), 3);
assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK");
let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle");
drop(commit);
caller.abort();
assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled());
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain)
.await
.expect("ordinary tail timeout stays bounded after ACK")
.expect("tail owner must not panic")
.expect("successful ACK keeps its convergence result");
assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback");
let pending = ctx.namespace_commits_pending();
let alive = owner_probe.upgrade().is_some();
let generation = ctx.namespace_commit_generation();
for disk in disks.iter().flatten().take(3) {
let stored = disk
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("all ACK voters keep the new object after caller cancellation");
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
drop(release_tx);
let lease = tokio::time::timeout(
Duration::from_secs(5),
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination),
)
.await
.expect("late physical tail drains");
drop(lease);
tokio::time::timeout(Duration::from_secs(5), async {
while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("late physical tail must release namespace ownership");
for dir in &dirs {
let stored = reopen_local_disk(dir)
.await
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("successful ACK remains committed on every disk after late publication");
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
assert!(
pending && alive,
"physical ACK tail must retain namespace ownership after the coordinator exits"
);
assert!(!ctx.namespace_commits_pending());
assert!(owner_probe.upgrade().is_none());
assert!(ctx.namespace_commit_generation() > generation);
assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback");
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for (allow_early_ack, cancel_caller, object) in [
(false, false, "rollback-barrier-object"),
(false, true, "rollback-barrier-cancelled"),
(true, false, "rollback-barrier-early-object"),
(true, true, "rollback-barrier-early-cancelled"),
] {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let bucket = "rename-rollback-barrier";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be staged");
}
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
let receipt = RenameRollbackReceipt::default();
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(object, 4, "new-etag"),
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"),
}
})
.await
.expect("undo must reach its disk barrier");
assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success");
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), 1);
if cancel_caller {
drop(rename);
assert!(ctx.namespace_commits_pending(), "caller cancellation must not retire pending undo work");
assert_eq!(ctx.namespace_commit_generation(), 1);
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while receipt.0.get().is_none() || ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled caller must not cancel rollback accounting");
} else {
barrier.release();
assert!(rename.await.is_err());
}
assert!(
!ctx.namespace_commits_pending(),
"the completed rollback must release its namespace ownership"
);
assert_eq!(ctx.namespace_commit_generation(), 2);
assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt");
for dir in dirs.iter().skip(1) {
let reopened = reopen_local_disk(dir).await;
let restored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("old version must remain readable after caller cancellation");
assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_children_keep_namespace_ownership_after_coordinator_panic() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for (allow_early_ack, object) in [
(false, "rollback-coordinator-panic"),
(true, "rollback-coordinator-panic-early"),
] {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let bucket = "rename-rollback-coordinator-panic";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be staged");
}
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _rollback_fault =
rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::RollbackCoordinatorPanic);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
let receipt = RenameRollbackReceipt::default();
let result = tokio::time::timeout(
BARRIER_PAUSE_GUARD,
SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(object, 4, "new-etag"),
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
),
)
.await
.expect("coordinator failure must return without waiting for detached undo tasks");
assert!(result.is_err());
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("detached undo must reach its disk barrier");
assert!(
receipt.is_incomplete(),
"coordinator failure must preserve indeterminate recovery evidence"
);
assert!(ctx.namespace_commits_pending(), "the paused child must retain namespace ownership");
assert_eq!(ctx.namespace_commit_generation(), 1);
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("completed undo children must release their namespace ownership");
assert_eq!(ctx.namespace_commit_generation(), 2);
for dir in &dirs {
let reopened = reopen_local_disk(dir).await;
let restored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("old version must remain readable after rollback coordinator failure");
assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_early_ack_strict_quorum_failure_rolls_back_fresh_after_reopen() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
const DISKS: usize = 4;
let bucket = "rename-early-ack-strict-rollback-bucket";
let object = "rename-early-ack-strict-rollback-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "early-strict-rollback-etag");
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
assert!(
SetDisks::rename_data_owned(&disks, (RUSTFS_META_TMP_BUCKET, "source"), file_infos, (bucket, object), 4, true,)
.await
.is_err(),
"three successful disks must fail an early-ACK strict write quorum of four"
);
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let read = reopened.read_version("", bucket, object, "", &ReadOptions::default()).await;
assert!(
matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"disk {idx} must not expose a fresh object after early-ACK strict rollback and reopen: {read:?}"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_commits_fresh_object_when_tail_disk_fails_after_write_quorum() {
const DISKS: usize = 4;
let bucket = "rename-tail-failure-fresh-bucket";
let object = "rename-tail-failure-fresh-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut file_infos = rename_commit_fileinfos(object, DISKS, "fresh-etag");
file_infos[3] = FileInfo::default();
SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3)
.await
.expect("three successful disks must satisfy write quorum despite one tail failure");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let read = reopened.read_version("", bucket, object, "", &ReadOptions::default()).await;
if idx < 3 {
read.unwrap_or_else(|err| panic!("quorum disk {idx} must contain the fresh commit after reopen: {err:?}"));
} else {
assert!(
matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"failed tail disk must not expose a partial fresh commit after reopen: {read:?}"
);
}
}
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_overwrite_tail_failure_preserves_old_tail_version_after_reopen() {
const DISKS: usize = 4;
let bucket = "rename-tail-failure-overwrite-bucket";
let object = "rename-tail-failure-overwrite-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be written before overwrite");
}
let mut file_infos = rename_commit_fileinfos(object, DISKS, "new-etag");
file_infos[3] = FileInfo::default();
SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3)
.await
.expect("three successful disks must satisfy overwrite quorum despite one tail failure");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must have a readable version after reopen: {err:?}"));
let expected_etag = if idx < 3 { "new-etag" } else { "old-etag" };
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some(expected_etag),
"disk {idx} must keep the correct overwrite visibility after reopen"
);
}
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_strict_quorum_failure_rolls_back_fresh_object_after_reopen() {
let _mode = durability_mode_override::set(DurabilityMode::Strict);
const DISKS: usize = 4;
let bucket = "rename-strict-rollback-fresh-bucket";
let object = "rename-strict-rollback-fresh-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut file_infos = rename_commit_fileinfos(object, DISKS, "fresh-rollback-etag");
file_infos[3] = FileInfo::default();
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
ctx.set_scanner_publication_state(false);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK);
let rename = SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
false,
RenameDataFenceOptions::new(4, None).with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
);
let control = async {
barrier.wait_until_paused().await;
assert!(ctx.namespace_commits_pending(), "rollback must retain namespace publication ownership");
assert!(ctx.scanner_publication_state_allowed(), "rollback must not disable namespace walks");
assert_eq!(ctx.namespace_commit_generation(), 1);
barrier.release();
};
let (result, ()) = tokio::time::timeout(BARRIER_PAUSE_GUARD, async { tokio::join!(rename, control) })
.await
.expect("rename rollback must reach its barrier and finish after release");
assert_eq!(
result.err(),
Some(DiskError::ErasureWriteQuorum),
"three successful disks must fail a strict write quorum of four"
);
assert!(!ctx.namespace_commits_pending());
assert!(ctx.scanner_publication_state_allowed());
assert_eq!(ctx.namespace_commit_generation(), 2);
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let read = reopened.read_version("", bucket, object, "", &ReadOptions::default()).await;
assert!(
matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"disk {idx} must not expose a fresh object after strict quorum rollback and reopen: {read:?}"
);
}
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_data_strict_quorum_failure_restores_overwrite_after_reopen() {
let _mode = durability_mode_override::set(DurabilityMode::Strict);
const DISKS: usize = 4;
let bucket = "rename-strict-rollback-overwrite-bucket";
let object = "rename-strict-rollback-overwrite-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.metadata.insert("etag".to_string(), "old-rollback-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be written before overwrite rollback test");
}
let mut file_infos = rename_commit_fileinfos(object, DISKS, "new-rollback-etag");
file_infos[3] = FileInfo::default();
SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 4)
.await
.expect_err("three successful disks must fail a strict overwrite quorum of four");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must keep old metadata after strict rollback: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("old-rollback-etag"),
"disk {idx} must restore the old overwrite target after strict rollback and reopen"
);
}
}
#[tokio::test]
async fn commit_cleanup_fanout_barrier_pauses_background_deletes() {
let bucket = "cleanup-barrier-bucket";
let object = "cleanup-barrier-object";
let old_data_dir = "11111111-1111-1111-1111-111111111111";
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
let path = format!("{object}/{old_data_dir}/part.1");
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&path, b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[(&path, b"two".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone())], 1).await;
let disks = [Some(disk1.clone()), Some(disk2.clone())];
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
let cleanup_fut = set.commit_rename_data_dir(&disks, bucket, object, old_data_dir, committed_data_dir, 2);
let control_fut = async {
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("cleanup fan-out must reach the armed cleanup barrier");
assert!(tracker.running() >= 1, "a background cleanup delete must still be in flight while paused");
barrier.release();
};
let (cleanup, ()) = tokio::join!(cleanup_fut, control_fut);
assert_eq!(tracker.running(), 0, "no background cleanup task may remain after drain");
assert_eq!(cleanup.attempted, 2);
assert_eq!(cleanup.reclaimed, 2, "the real old-data-dir must still be reclaimed after release");
drop((disk1, disk2));
}
#[tokio::test]
async fn commit_cleanup_reports_and_releases_deferred_snapshot_data_dirs() {
let bucket = "cleanup-lease-bucket";
let object = "cleanup-lease-object";
let old_data_dir = "11111111-1111-1111-1111-111111111111";
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
let data_dir_path = format!("{object}/{old_data_dir}");
let shard_path = format!("{data_dir_path}/part.1");
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&shard_path, b"one".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone())], 0).await;
let lease = disk1
.acquire_snapshot_lease(bucket, &data_dir_path)
.await
.expect("snapshot lease should be acquired before cleanup");
let cleanup = set
.commit_rename_data_dir(&[Some(disk1.clone())], bucket, object, old_data_dir, committed_data_dir, 1)
.await;
assert_eq!(cleanup.attempted, 1);
assert_eq!(cleanup.reclaimed, 0);
assert_eq!(cleanup.deferred, 1);
assert!(cleanup.unreclaimed_disks.is_empty());
disk1
.read_all(bucket, &shard_path)
.await
.expect("deferred cleanup must leave later shard opens available");
disk1
.release_snapshot_lease(bucket, &data_dir_path, lease)
.await
.expect("final lease release should reclaim the old data directory");
assert!(matches!(disk1.read_all(bucket, &shard_path).await, Err(DiskError::FileNotFound)));
}
#[tokio::test]
async fn barrier_and_task_tracker_isolate_by_object() {
const DISKS: usize = 3;
let bucket = "barrier-iso-bucket";
let observed_object = "iso-observed-object";
let other_object = "iso-other-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
let tracker = rename_fanout_barrier::observe_tasks(observed_object);
let _barrier = rename_fanout_barrier::arm(observed_object, 0, rename_fanout_barrier::PHASE_RENAME);
let file_infos = rename_barrier_fileinfos(other_object, DISKS);
let _ = tokio::time::timeout(
BARRIER_PAUSE_GUARD,
SetDisks::rename_data(&disks, bucket, other_object, &file_infos, bucket, other_object, DISKS - 1),
)
.await
.expect("fan-out for an unarmed object must not pause");
assert_eq!(
tracker.running(),
0,
"another object's fan-out must not accrue tracked tasks for the observed object"
);
drop(dirs);
}
#[tokio::test]
async fn multipart_codec_streaming_reader_zero_buffer_is_noop() {
let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
let mut reader = MultipartCodecStreamingReader::new(vec![Box::new(reader)]);
let mut out = [];
let read = reader
.read(&mut out)
.await
.expect("zero-length read should not poll inner readers");
assert_eq!(read, 0);
assert_eq!(reader.readers.len(), 1);
}
#[test]
fn metadata_fanout_observation_classifies_invalid_and_ignored_results() {
let invalid = MetadataFanoutObservation::from_file_info(&FileInfo::default(), Duration::from_millis(7));
assert_eq!(invalid.outcome, GET_METADATA_RESPONSE_ERROR);
assert!(!invalid.valid);
assert!(!invalid.ignored);
let ignored = MetadataFanoutObservation::from_error(&DiskError::DiskNotFound, Duration::from_millis(9));
assert_eq!(ignored.outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
assert!(!ignored.valid);
assert!(ignored.ignored);
let corrupt = MetadataFanoutObservation::from_error(&DiskError::FileCorrupt, Duration::from_millis(11));
assert_eq!(corrupt.outcome, GET_METADATA_RESPONSE_CORRUPT);
assert!(!corrupt.ignored);
}
#[test]
fn metadata_fanout_diagnostics_reports_counts_and_latency_edges() {
let diagnostics = MetadataFanoutDiagnostics::new(
Duration::from_millis(40),
vec![
MetadataFanoutObservation {
outcome: GET_METADATA_RESPONSE_VALID,
elapsed: Duration::from_millis(30),
valid: true,
ignored: false,
},
MetadataFanoutObservation {
outcome: GET_METADATA_RESPONSE_IGNORED,
elapsed: Duration::from_millis(10),
valid: false,
ignored: true,
},
MetadataFanoutObservation {
outcome: GET_METADATA_RESPONSE_ERROR,
elapsed: Duration::from_millis(20),
valid: false,
ignored: false,
},
],
);
assert_eq!(diagnostics.total_responses(), 3);
assert_eq!(diagnostics.valid_responses(), 1);
assert_eq!(diagnostics.ignored_responses(), 1);
assert_eq!(diagnostics.non_valid_responses(), 2);
assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
assert_eq!(diagnostics.quorum_candidate_latency(0), Some(Duration::ZERO));
assert_eq!(diagnostics.quorum_candidate_latency(1), Some(Duration::from_millis(30)));
assert_eq!(diagnostics.quorum_candidate_latency(2), None);
}
#[test]
fn metadata_quorum_accumulator_counts_invalid_metadata_and_ignored_errors() {
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
accumulator.observe_file_info(&FileInfo::default());
accumulator.observe_error(&DiskError::DiskNotFound);
assert_eq!(accumulator.hard_errors, 1);
assert_eq!(accumulator.ignored_errors, 1);
assert_eq!(accumulator.final_miss_reason(), GET_METADATA_EARLY_STOP_REASON_ERROR);
}
#[test]
fn metadata_quorum_accumulator_early_stops_on_one_delete_marker_majority() {
let marker = metadata_test_delete_marker("object", Uuid::new_v4(), OffsetDateTime::now_utc());
let mut accumulator = MetadataQuorumAccumulator::new(6, 3, true);
for _ in 0..4 {
accumulator.observe_file_info(&marker);
}
assert_eq!(accumulator.default_write_quorum(), 4);
assert_eq!(accumulator.delete_marker_votes, 4);
assert_eq!(
accumulator.early_stop_decision(),
Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
})
);
}
#[test]
fn metadata_quorum_accumulator_does_not_combine_distinct_delete_markers() {
let now = OffsetDateTime::now_utc();
let first = metadata_test_delete_marker("object", Uuid::new_v4(), now);
let second = metadata_test_delete_marker("object", Uuid::new_v4(), now + time::Duration::seconds(1));
let third = metadata_test_delete_marker("object", Uuid::new_v4(), now + time::Duration::seconds(2));
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
accumulator.observe_file_info(&first);
accumulator.observe_file_info(&second);
accumulator.observe_file_info(&third);
assert_eq!(accumulator.delete_marker_votes, 1);
assert_eq!(accumulator.delete_marker_candidates.len(), 3);
assert_eq!(accumulator.early_stop_decision(), None);
accumulator.observe_file_info(&second);
accumulator.observe_file_info(&second);
assert_eq!(accumulator.delete_marker_votes, 3);
assert!(accumulator.early_stop_decision().is_some());
}
#[test]
fn metadata_quorum_accumulator_candidate_latest_quorum_handles_zero_parity_and_invalid_candidates() {
let accumulator = MetadataQuorumAccumulator::new(4, 0, true);
let candidate = metadata_test_fileinfo("object");
assert_eq!(accumulator.candidate_latest_quorum(&candidate), Some(4));
assert_eq!(accumulator.missing_response_quorum(), 4);
let accumulator = MetadataQuorumAccumulator::new(4, 2, true);
let mut deleted = candidate.clone();
deleted.deleted = true;
assert_eq!(accumulator.candidate_latest_quorum(&deleted), Some(3));
let marker = metadata_test_delete_marker("object", Uuid::new_v4(), OffsetDateTime::now_utc());
assert_eq!(accumulator.candidate_latest_quorum(&marker), None);
let mut empty = candidate.clone();
empty.size = 0;
assert_eq!(accumulator.candidate_latest_quorum(&empty), None);
let mut impossible_parity = candidate;
impossible_parity.erasure.parity_blocks = 4;
assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None);
}
#[test]
fn metadata_quorum_accumulator_tracks_mapped_shards_and_requires_a_reserve() {
let version_id = Uuid::new_v4();
let data_dir = Uuid::new_v4();
let base = valid_metadata_fanout_fileinfo("bucket", "object", version_id, data_dir, OffsetDateTime::now_utc());
let distribution = base.erasure.distribution.clone();
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
for (disk_index, &erasure_index) in distribution.iter().take(2).enumerate() {
let mut file_info = base.clone();
file_info.erasure.index = erasure_index;
accumulator.observe_file_info_at(disk_index, &file_info);
}
assert!(
!accumulator.candidate_has_read_reserve(),
"data quorum without parity reserve must not early-stop"
);
let mut mismatched = base.clone();
mismatched.erasure.index = distribution[3];
accumulator.observe_file_info_at(2, &mismatched);
assert!(
!accumulator.candidate_has_read_reserve(),
"mapped index mismatch must not count as a reserve"
);
let mut reserve = base;
reserve.erasure.index = distribution[2];
accumulator.observe_file_info_at(2, &reserve);
assert!(
accumulator.candidate_has_read_reserve(),
"one matching reserve shard should complete the read reserve"
);
}
#[test]
fn metadata_quorum_accumulator_treats_invalid_default_parity_as_full_fanout() {
let accumulator = MetadataQuorumAccumulator::new(2, 2, true);
assert_eq!(accumulator.default_write_quorum(), 2);
assert_eq!(accumulator.missing_response_quorum(), 2);
assert!(accumulator.can_still_reach_early_stop_with_pending(2));
assert!(!accumulator.can_still_reach_early_stop_with_pending(1));
}
#[test]
fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
assert!(!is_confirmed_missing_part_error(None));
assert!(is_confirmed_missing_part_error(Some("file not found")));
assert!(is_confirmed_missing_part_error(Some("No such file or directory")));
assert!(is_confirmed_missing_part_error(Some("Specified part could not be found")));
assert!(is_confirmed_missing_part_error(Some("part.7 not found")));
assert!(!is_confirmed_missing_part_error(Some("part.7 missing")));
assert!(!is_confirmed_missing_part_error(Some("permission denied")));
}
#[test]
fn resolve_read_part_handles_mismatched_and_transient_responses_without_false_missing() {
let responses = vec![
Some(Vec::new()),
Some(vec![read_part_test_error(1, "permission denied")]),
None,
];
let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.expect_err("mismatched and transient responses must not be treated as confirmed missing");
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[test]
fn resolve_read_part_accepts_alternate_missing_error_markers() {
let responses = vec![
Some(vec![read_part_test_error(1, "Specified part could not be found")]),
Some(vec![read_part_test_error(1, "part.1 not found")]),
Some(vec![read_part_test_part(1, "stale-etag")]),
];
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.expect("alternate missing markers should satisfy missing quorum");
assert_eq!(part.number, 1);
assert_eq!(part.error.as_deref(), Some("part.1 not found"));
assert!(part.etag.is_empty());
}
#[test]
fn resolve_read_part_requires_layout_fields_to_reach_quorum() {
let mut valid = read_part_test_part(1, "winner");
valid.size = 100;
valid.actual_size = 90;
let mut wrong_etag = valid.clone();
wrong_etag.etag = "loser".to_string();
let mut wrong_number = valid.clone();
wrong_number.number = 2;
let mut wrong_size = valid.clone();
wrong_size.size = 50;
let mut wrong_actual_size = valid.clone();
wrong_actual_size.actual_size = 40;
for (field, corrupted) in [
("etag", wrong_etag),
("number", wrong_number),
("size", wrong_size),
("actual_size", wrong_actual_size),
] {
let responses = vec![Some(vec![corrupted]), Some(vec![valid.clone()]), Some(vec![valid.clone()])];
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.unwrap_or_else(|err| panic!("{field} mismatch must not defeat layout quorum: {err}"));
assert_eq!(part.etag, "winner", "{field}");
assert_eq!(part.number, 1, "{field}");
assert_eq!(part.size, 100, "{field}");
assert_eq!(part.actual_size, 90, "{field}");
}
}
#[test]
fn resolve_read_part_diagnostic_branch_keeps_read_quorum_error() {
temp_env::with_var("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"), || {
let responses = vec![
Some(Vec::new()),
None,
Some(vec![read_part_test_error(1, "permission denied")]),
];
let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.expect_err("diagnostic logging must not change the read-quorum result");
assert_eq!(err, DiskError::ErasureReadQuorum);
});
}
#[tokio::test]
async fn shard_read_costs_for_empty_disk_set_are_empty() {
assert!(shard_read_costs_for_disks(&[]).is_empty());
}
#[test]
fn shard_read_cost_for_endpoint_and_missing_disk_cover_all_cost_classes() {
let same_node_hosts = vec!["node-a:9000".to_string()];
assert_eq!(shard_read_cost_for_disk(None, &same_node_hosts), ShardReadCost::Unknown);
assert_eq!(shard_read_cost_for_endpoint(true, "", &same_node_hosts), ShardReadCost::Local);
assert_eq!(
shard_read_cost_for_endpoint(false, "node-a:9000", &same_node_hosts),
ShardReadCost::SameNode
);
assert_eq!(
shard_read_cost_for_endpoint(false, "node-b:9000", &same_node_hosts),
ShardReadCost::Remote
);
assert!(local_endpoint_hosts_for_shard_costs().is_empty());
}
#[test]
#[serial_test::serial]
fn bitrot_reader_setup_tracks_strategy_counters_and_deferred_readers() {
temp_env::with_var(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, None::<&str>, || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::ReadQuorum, false),
BitrotReaderSetupStrategy::DataBlocksFirst
));
});
temp_env::with_var(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("false"), || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::ReadQuorum, false),
BitrotReaderSetupStrategy::AllShards
));
});
temp_env::with_var(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true"), || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::ReadQuorum, false),
BitrotReaderSetupStrategy::DataBlocksFirst
));
});
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, Some("true"), || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::VerifyReconstruction, false),
BitrotReaderSetupStrategy::DataBlocksFirst
));
});
assert_eq!(BitrotReaderSetupMode::ReadQuorum.as_str(), "read_quorum");
assert_eq!(BitrotReaderSetupMode::VerifyReconstruction.as_str(), "verify_reconstruction");
assert_eq!(BitrotReaderSetupStrategy::AllShards.as_str(), "all_shards");
assert_eq!(BitrotReaderSetupStrategy::DataBlocksFirst.as_str(), "data_blocks_first");
assert_eq!(BitrotReaderSetupStrategy::DataBlocksOnly.as_str(), "data_blocks_only");
let mut setup = BitrotReaderSetup::new(4);
assert_eq!(setup.scheduled_shards(), 0);
assert!(setup.mark_scheduled(0));
assert!(!setup.mark_scheduled(0));
assert_eq!(setup.scheduled_shards(), 1);
assert_eq!(setup.pending_scheduled_shards(), 1);
assert_eq!(setup.available_shards(), 0);
assert_eq!(setup.completed_failed_shards(), 0);
assert_eq!(setup.reconstruction_verification_target(3, 2), 3);
assert!(!setup.has_setup_quorum(3, 2, BitrotReaderSetupMode::ReadQuorum));
assert!(!setup.data_shards_attempted(3));
assert_eq!(setup.scheduling_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
setup.apply_reader_result(0, Ok(Some(test_object_bitrot_reader())));
setup.apply_reader_result(1, Ok(None));
setup.apply_reader_result(2, Err(DiskError::FileCorrupt));
assert_eq!(setup.attempted_shards(), 3);
assert_eq!(setup.pending_scheduled_shards(), 0);
assert_eq!(setup.available_shards(), 1);
assert_eq!(setup.available_data_shards(3), 1);
assert_eq!(setup.completed_failed_shards(), 2);
assert!(setup.data_shards_attempted(3));
assert_eq!(setup.reconstruction_verification_target(3, 2), 3);
assert_eq!(setup.setup_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
assert_eq!(setup.scheduling_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
let mut verification_setup = BitrotReaderSetup::new(4);
verification_setup.apply_reader_result(0, Ok(Some(test_object_bitrot_reader())));
verification_setup.apply_reader_result(1, Ok(Some(test_object_bitrot_reader())));
verification_setup.apply_reader_result(2, Err(DiskError::FileCorrupt));
verification_setup.apply_reader_result(3, Ok(Some(test_object_bitrot_reader())));
assert_eq!(verification_setup.reconstruction_verification_target(3, 2), 4);
assert_eq!(verification_setup.setup_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 4);
assert!(verification_setup.has_setup_quorum(3, 2, BitrotReaderSetupMode::ReadQuorum));
let (deferred_reader, stripe_handle) = test_deferred_object_bitrot_reader();
setup.retain_deferred_reader(3, deferred_reader, stripe_handle);
assert_eq!(setup.deferred_shards(), 1);
assert!(setup.readers[3].is_some());
assert!(setup.errors[3].is_none());
}
#[tokio::test]
async fn write_unique_file_info_reverts_metadata_when_write_quorum_fails() {
let bucket = "write-unique-bucket";
let object = "object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
let files = vec![fi.clone(), fi];
let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await;
assert!(result.is_err(), "missing disk must prevent the requested write quorum");
assert!(
matches!(
disk.read_all(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE])).await,
Err(DiskError::FileNotFound)
),
"successful metadata write must be reverted when quorum is not reached"
);
}
#[tokio::test]
async fn write_unique_file_info_rollback_preserves_existing_reconciled_version() {
let bucket = "write-unique-existing";
let object = "object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut original = metadata_test_fileinfo(object);
original.version_id = Some(Uuid::from_u128(1));
original.data_dir = Some(Uuid::from_u128(3));
original.mod_time = Some(OffsetDateTime::now_utc());
original.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
original.transition_tier = "WARM".to_string();
original.transitioned_objname = "remote-original".to_string();
original.transition_version_state = rustfs_filemeta::TransitionVersionState::KnownDisabled;
rustfs_utils::http::insert_str(
&mut original.metadata,
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
"ab".repeat(32),
);
disk.write_metadata(bucket, bucket, object, original.clone())
.await
.expect("existing reconciled source");
let raw = disk
.read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
.await
.expect("original metadata");
let before = FileMeta::load(&raw).unwrap().find_version(original.version_id).unwrap().1;
let mut added = original.clone();
added.version_id = Some(Uuid::from_u128(2));
let result =
SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &[added.clone(), added], 2)
.await;
assert!(result.is_err());
let raw = disk
.read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
.await
.expect("preserved xl.meta");
let after = FileMeta::load(&raw).expect("preserved metadata");
assert_eq!(
after
.find_version(original.version_id)
.expect("original source survives rollback")
.1,
before
);
assert!(after.find_version(Some(Uuid::from_u128(2))).is_err());
}
#[tokio::test]
async fn update_object_meta_handles_empty_metadata_and_missing_quorum() {
let set = io_primitives_test_set(vec![None, None], 1).await;
let mut empty = metadata_test_fileinfo("object");
empty.metadata.clear();
set.update_object_meta("bucket", "object", empty, &[None, None])
.await
.expect("empty metadata update without replacement should be a no-op");
let mut with_metadata = metadata_test_fileinfo("object");
with_metadata
.metadata
.insert("x-amz-meta-test".to_string(), "value".to_string());
let result = set.update_object_meta("bucket", "object", with_metadata, &[None, None]).await;
assert!(result.is_err(), "missing disks must prevent metadata write quorum");
}
#[tokio::test]
async fn tier_overwrite_cleanup_rejects_unreadable_disk_despite_metadata_quorum() {
let bucket = "tier-unreadable-disk";
let object = "object";
let mut dirs = Vec::new();
let mut disks = Vec::new();
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
for index in 1..=3 {
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
fi.erasure.index = index;
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("seed metadata quorum");
dirs.push(dir);
disks.push(Some(disk));
}
disks.push(None);
let set = io_primitives_test_set(disks, 2).await;
assert!(
set.load_file_info_versions_exact(bucket, object).await.is_err(),
"exact reads must preserve release's unreadable-replica fence"
);
assert!(
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
"unreadable replica may still reference the old remote object"
);
}
#[tokio::test]
async fn tier_overwrite_cleanup_rejects_minority_metadata_in_an_absent_set() {
let bucket = "tier-minority-metadata";
let object = "object";
let mut dirs = Vec::new();
let mut disks = Vec::new();
for index in 1..=4 {
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
if index == 1 {
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi)
.await
.expect("seed minority metadata");
}
dirs.push(dir);
disks.push(Some(disk));
}
let set = io_primitives_test_set(disks, 2).await;
assert!(
set.load_file_info_versions_exact(bucket, object).await.is_err(),
"exact reads must preserve release's minority-ownership fence"
);
assert!(
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
"absence on a majority cannot prove this physical set has no remote reference"
);
}
#[tokio::test]
async fn load_file_info_versions_exact_returns_versions_from_read_quorum() {
let bucket = "exact-versions-bucket";
let object = "exact-object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut fi = metadata_test_fileinfo(object);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("metadata should be written");
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let versions = set
.load_file_info_versions_exact(bucket, object)
.await
.expect("exact version load should succeed")
.expect("exact version load should find metadata");
assert_eq!(versions.versions.len(), 1);
assert_eq!(versions.versions[0].version_id, fi.version_id);
assert_eq!(versions.versions[0].name, object);
}
#[tokio::test]
async fn load_file_info_versions_exact_encodes_directory_key_but_returns_logical_name() {
let bucket = "exact-directory-versions-bucket";
let object = "prefix/directory/";
let disk_object = rustfs_utils::path::encode_dir_object(object);
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut fi = metadata_test_fileinfo(object);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, disk_object.as_str(), fi.clone())
.await
.expect("directory metadata should be written under the encoded key");
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let versions = set
.load_file_info_versions_exact(bucket, object)
.await
.expect("exact directory version load should succeed")
.expect("exact directory version load should find metadata");
assert_eq!(versions.name, object);
assert_eq!(versions.versions.len(), 1);
assert_eq!(versions.versions[0].name, object);
assert_eq!(versions.versions[0].version_id, fi.version_id);
}
#[tokio::test]
async fn load_file_info_versions_exact_rejects_transitioned_duplicate_parts() {
let bucket = "exact-versions-bucket";
let object = "poisoned-transitioned-object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut file_info = metadata_test_fileinfo(object);
file_info.version_id = Some(Uuid::new_v4());
file_info.mod_time = Some(OffsetDateTime::now_utc());
file_info.transition_status = TRANSITION_COMPLETE.to_string();
file_info.transitioned_objname = "remote/object".to_string();
file_info.transition_tier = "WARM".to_string();
file_info.parts.push(file_info.parts[0].clone());
write_raw_file_info_unchecked(&disk, bucket, object, file_info).await;
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let err = set
.load_file_info_versions_exact(bucket, object)
.await
.expect_err("exact loader must reject metadata that would poison decommission");
assert!(err.to_string().contains("validation failed"), "unexpected error: {err}");
}
#[tokio::test]
async fn load_file_info_versions_exact_rejects_default_like_delete_marker() {
let bucket = "exact-versions-bucket";
let object = "forged-delete-marker";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let forged_version = rustfs_filemeta::FileMetaVersion {
version_type: rustfs_filemeta::VersionType::Delete,
delete_marker: Some(rustfs_filemeta::MetaDeleteMarker {
version_id: Some(Uuid::new_v4()),
mod_time: None,
..Default::default()
}),
write_version: 1,
..Default::default()
};
let mut forged_meta = FileMeta::new();
forged_meta
.versions
.push(FileMetaShallowVersion::try_from(forged_version).expect("forged marker body should encode"));
write_raw_file_meta_unchecked(&disk, bucket, object, forged_meta).await;
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let err = set
.load_file_info_versions_exact(bucket, object)
.await
.expect_err("default-like delete marker must be rejected at the exact loader boundary");
assert!(err.to_string().contains("exact object versions decode failed"), "unexpected error: {err}");
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn commit_rename_data_dir_reclaims_old_data_dir_and_reports_receipt() {
use rustfs_object_capacity::capacity_scope::drain_global_dirty_scopes;
let bucket = "commit-rename-bucket";
let object = "object";
let old_data_dir = "11111111-1111-1111-1111-111111111111";
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
let path = format!("{object}/{old_data_dir}/part.1");
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&path, b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[(&path, b"two".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone())], 1).await;
let disks = [Some(disk1.clone()), Some(disk2.clone())];
let expected = capacity_scope_from_disks(&disks).disks.into_iter().collect::<HashSet<_>>();
let _ = drain_global_dirty_scopes();
let cleanup = set
.commit_rename_data_dir_and_mark_capacity(&disks, bucket, object, old_data_dir, committed_data_dir, 2)
.await;
assert_eq!(cleanup.attempted, 2);
assert_eq!(cleanup.reclaimed, 2);
assert!(!cleanup.has_residue());
assert!(matches!(disk1.read_all(bucket, &path).await, Err(DiskError::FileNotFound)));
assert!(matches!(disk2.read_all(bucket, &path).await, Err(DiskError::FileNotFound)));
let marked = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
assert!(
expected.is_subset(&marked),
"old-data cleanup must mark capacity before callers can await residue reporting"
);
let missing = set
.commit_rename_data_dir(&[None, None], bucket, object, old_data_dir, committed_data_dir, 1)
.await;
assert_eq!(missing.attempted, 0);
assert_eq!(missing.reclaimed, 0);
assert!(!missing.has_residue(), "missing disk slots must not be counted as cleanup residue");
assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
}
#[tokio::test]
async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
let bucket = "delete-prefix-bucket";
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await;
let (_dir3, disk3) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"three".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), Some(disk3.clone()), None], 2).await;
set.delete_prefix(bucket, "prefix")
.await
.expect("three successful disks should meet a four-disk write quorum");
for disk in [disk1, disk2, disk3] {
assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound)));
}
}
#[tokio::test]
async fn delete_prefix_counts_confirmed_absence_toward_quorum() {
let bucket = "delete-prefix-confirmed-absence";
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await;
let (_dir3, disk3) = read_multiple_test_disk(bucket, &[]).await;
let (_dir4, disk4) = read_multiple_test_disk(bucket, &[]).await;
disk3
.delete_volume(bucket, true)
.await
.expect("third disk bucket should be absent");
disk4
.delete_volume(bucket, true)
.await
.expect("fourth disk bucket should be absent");
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), Some(disk3), Some(disk4)], 2).await;
set.delete_prefix(bucket, "prefix")
.await
.expect("successful deletes and confirmed absence should jointly meet quorum");
for disk in [disk1, disk2] {
assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound)));
}
}
#[test]
fn delete_prefix_result_reduction_preserves_existing_volume_evidence() {
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![
Err(DiskError::FileNotFound),
Err(DiskError::FileNotFound),
Err(DiskError::FileNotFound),
Err(DiskError::DiskNotFound),
],
3,
),
Ok(())
);
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![
Err(DiskError::FileNotFound),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
],
3,
),
Ok(())
);
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![
Ok(()),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
],
3,
),
Ok(())
);
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
Err(DiskError::VolumeNotFound),
],
3,
),
Err(DiskError::VolumeNotFound)
);
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![Ok(()), Ok(()), Err(DiskError::DiskNotFound), Err(DiskError::DiskNotFound)],
3,
),
Err(DiskError::ErasureWriteQuorum)
);
assert_eq!(
SetDisks::reduce_delete_prefix_results(
vec![
Ok(()),
Err(DiskError::FileAccessDenied),
Err(DiskError::FileAccessDenied),
Err(DiskError::FileAccessDenied),
],
3,
),
Err(DiskError::FileAccessDenied)
);
}
#[tokio::test]
async fn delete_prefix_fails_at_quorum_minus_one() {
let bucket = "delete-prefix-quorum-minus-one";
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), None, None], 2).await;
let err = set
.delete_prefix(bucket, "prefix")
.await
.expect_err("two successful disks must not meet a four-disk write quorum");
assert_eq!(err, DiskError::ErasureWriteQuorum);
for disk in [disk1, disk2] {
assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound)));
}
}
#[tokio::test]
async fn delete_prefix_fails_when_all_disk_slots_are_missing() {
let set = io_primitives_test_set(vec![None, None, None, None], 2).await;
let err = set
.delete_prefix("delete-prefix-offline", "prefix")
.await
.expect_err("an entirely offline set must not report a successful deletion");
assert_eq!(err, DiskError::ErasureWriteQuorum);
}
#[tokio::test]
async fn delete_if_dangling_respects_recent_write_grace_without_deleting_metadata() {
let bucket = "dangling-grace-bucket";
let object = "object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let set = io_primitives_test_set(vec![Some(disk.clone()), None, None], 1).await;
let mut fi = FileInfo::new(object, 2, 1);
fi.volume = bucket.to_string();
fi.name = object.to_string();
fi.size = 1;
fi.erasure.index = 1;
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("metadata should be written before dangling check");
let err = set
.delete_if_dangling(
bucket,
object,
&[fi, FileInfo::default(), FileInfo::default()],
&[None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)],
&HashMap::new(),
ObjectOptions::default(),
)
.await
.expect_err("recent dangling metadata must stay protected by grace");
let message = err.to_string();
assert!(
message.contains(HEAL_DANGLING_DELETE_GRACE_MESSAGE),
"grace-protected dangling cleanup must explain the deferred delete: {message}"
);
assert!(
message.contains("retry_after_secs="),
"grace-protected dangling cleanup must include retry timing: {message}"
);
disk.read_all(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
.await
.expect("metadata should remain during dangling grace");
}
#[tokio::test]
async fn delete_if_dangling_returns_stale_metadata_when_all_slots_are_already_absent() {
let bucket = "dangling-delete-bucket";
let object = "object";
let set = io_primitives_test_set(vec![None, None, None], 1).await;
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc() - time::Duration::hours(2));
let deleted = set
.delete_if_dangling(
bucket,
object,
&[fi.clone(), FileInfo::default(), FileInfo::default()],
&[
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
],
&HashMap::new(),
ObjectOptions::default(),
)
.await
.expect("stale dangling metadata should pass when every delete target was already absent");
assert_eq!(deleted.name, object);
assert_eq!(deleted.mod_time, fi.mod_time);
}
#[tokio::test]
async fn delete_if_dangling_cleans_when_only_part_results_prove_invalid_metadata_is_dangling() {
let bucket = "dangling-invalid-meta-bucket";
let object = "object";
let set = io_primitives_test_set(vec![None, None, None, None], 1).await;
let mut data_errs_by_part = HashMap::new();
data_errs_by_part.insert(
1,
vec![
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_DISK_NOT_FOUND,
],
);
let deleted = set
.delete_if_dangling(
bucket,
object,
&[
FileInfo::default(),
FileInfo::default(),
FileInfo::default(),
FileInfo::default(),
],
&[
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::DiskNotFound),
],
&data_errs_by_part,
ObjectOptions::default(),
)
.await
.expect("invalid metadata should be cleanable when part results prove the object is dangling");
assert!(!deleted.is_valid());
}
#[tokio::test]
async fn read_multiple_files_returns_quorum_data_and_fails_closed_for_partial_file() {
let bucket = "read-multiple-bucket";
let prefix = "prefix";
let (_dir1, disk1) = read_multiple_test_disk(
bucket,
&[
("prefix/shared.txt", b"longer shared payload".as_slice()),
("prefix/partial.txt", b"only one disk".as_slice()),
],
)
.await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/shared.txt", b"short".as_slice())]).await;
let req = ReadMultipleReq {
bucket: bucket.to_string(),
prefix: prefix.to_string(),
files: vec!["shared.txt".to_string(), "partial.txt".to_string()],
max_size: 0,
metadata_only: false,
abort404: false,
max_results: 0,
};
let responses = SetDisks::read_multiple_files(&[Some(disk1), Some(disk2)], req, 2).await;
assert_eq!(responses.len(), 2);
assert!(responses[0].exists);
assert_eq!(responses[0].data, b"longer shared payload");
assert!(!responses[1].exists);
assert_eq!(responses[1].error, Error::ErasureReadQuorum.to_string());
}
#[tokio::test]
async fn read_multiple_files_returns_read_quorum_error_when_no_disk_can_answer() {
let req = ReadMultipleReq {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
files: vec!["missing.txt".to_string()],
max_size: 0,
metadata_only: false,
abort404: false,
max_results: 0,
};
let responses = SetDisks::read_multiple_files(&[None, None], req, 1).await;
assert_eq!(responses.len(), 1);
assert!(!responses[0].exists);
assert_eq!(responses[0].error, Error::ErasureReadQuorum.to_string());
}
#[tokio::test]
#[serial_test::serial]
async fn mrf_ingress_is_coalesced_independently_of_read_repair_reservations() {
let bucket = format!("mrf-intent-bucket-{}", Uuid::new_v4());
let object = format!("object-{}", Uuid::new_v4());
let mut receiver = rustfs_common::mrf_channel::init_mrf_channel().expect("first channel init in this binary");
rustfs_common::mrf_channel::set_mrf_delivery_enabled(true);
fn intent_submission<'a>(bucket: &'a str, object: &'a str) -> ReadRepairHealSubmission<'a> {
ReadRepairHealSubmission {
bucket,
object,
version_id: None,
pool_index: 9,
set_index: 9,
part_number: Some(1),
reason: "decode_error",
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, None)),
}
}
let mut metadata_submission = intent_submission(&bucket, &object);
metadata_submission.mrf_intent = None;
submit_read_repair_heal_with_submitter(metadata_submission, accepted_read_repair_submitter).await;
assert!(receiver.try_recv().is_err());
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
let first = receiver.try_recv().expect("first sighting must file exactly one MRF intent");
assert_eq!(*first.bucket, bucket);
assert_eq!(*first.object, object);
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
assert!(receiver.try_recv().is_err(), "duplicate sighting must not file another MRF intent");
let retry_object = format!("retry-object-{}", Uuid::new_v4());
rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
assert!(receiver.try_recv().is_err());
rustfs_common::mrf_channel::set_mrf_delivery_enabled(true);
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
let retried = receiver
.try_recv()
.expect("a heal reservation cannot suppress a previously rejected MRF intent");
assert_eq!(*retried.object, retry_object);
rustfs_common::mrf_channel::release_mrf_intent(&first);
rustfs_common::mrf_channel::release_mrf_intent(&retried);
rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
}
#[tokio::test]
async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
let object = format!("object-{}", Uuid::new_v4());
let key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
.await
.expect("first read-repair reservation should be accepted");
assert_eq!(key.version_id.as_deref(), Some("version-1"));
assert!(
reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
.await
.is_none()
);
release_read_repair_heal_reservation(&key).await;
let retry_key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
.await
.expect("released read-repair reservation should allow retry");
release_read_repair_heal_reservation(&retry_key).await;
}
#[test]
fn record_read_repair_dedup_counts_each_reason_separately() {
let recorder = crate::test_metrics::CapturingRecorder::default();
metrics::with_local_recorder(&recorder, || {
record_read_repair_dedup("duplicate");
record_read_repair_dedup("duplicate");
record_read_repair_dedup("policy_drop");
});
assert_eq!(
recorder.counter_value("rustfs_heal_read_repair_dedup_total", &[("reason", "duplicate")]),
2
);
assert_eq!(
recorder.counter_value("rustfs_heal_read_repair_dedup_total", &[("reason", "policy_drop")]),
1
);
}
#[tokio::test]
async fn reserve_read_repair_heal_prunes_oldest_entry_at_capacity() {
let bucket = format!("bucket-{}", Uuid::new_v4());
let first_object = format!("object-{}", Uuid::new_v4());
let first_key = reserve_read_repair_heal(&bucket, &first_object, None, 1, 1)
.await
.expect("first reservation should be accepted");
let mut keys = vec![first_key.clone()];
for index in 0..(READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES + 8) {
let object = format!("object-{index}-{}", Uuid::new_v4());
if let Some(key) = reserve_read_repair_heal(&bucket, &object, None, 1, 1).await {
keys.push(key);
}
}
let replaced_first = reserve_read_repair_heal(&bucket, &first_object, None, 1, 1).await;
for key in keys {
release_read_repair_heal_reservation(&key).await;
}
if let Some(key) = replaced_first.as_ref() {
release_read_repair_heal_reservation(key).await;
}
assert!(
replaced_first.is_some(),
"capacity pruning should evict the oldest read-repair reservation"
);
}
#[tokio::test]
#[serial_test::serial]
async fn submit_read_repair_heal_wrapper_releases_reservation_when_channel_unavailable() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal("bucket", &object, None, 4, 5, Some(7), "test").await;
for _ in 0..20 {
if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 4, 5).await {
release_read_repair_heal_reservation(&key).await;
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
panic!("unserviced heal channel must release the wrapper's read-repair dedup reservation");
}
#[tokio::test]
async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket: "bucket",
object: &object,
version_id: None,
pool_index: 1,
set_index: 2,
part_number: Some(1),
reason: "test",
mrf_intent: None,
},
failed_read_repair_submitter,
)
.await;
for _ in 0..20 {
if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 1, 2).await {
release_read_repair_heal_reservation(&key).await;
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
panic!("failed read-repair submission should release its dedup reservation");
}
#[tokio::test]
async fn submit_read_repair_heal_releases_reservation_after_not_admitted_response() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket: "bucket",
object: &object,
version_id: Some("version-1"),
pool_index: 2,
set_index: 3,
part_number: Some(2),
reason: "test",
mrf_intent: None,
},
dropped_read_repair_submitter,
)
.await;
for _ in 0..20 {
if let Some(key) = reserve_read_repair_heal("bucket", &object, Some("version-1"), 2, 3).await {
release_read_repair_heal_reservation(&key).await;
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
panic!("not-admitted read-repair submission should release its dedup reservation");
}
#[tokio::test]
async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket: "bucket",
object: &object,
version_id: None,
pool_index: 3,
set_index: 4,
part_number: None,
reason: "test",
mrf_intent: None,
},
accepted_read_repair_submitter,
)
.await;
assert!(reserve_read_repair_heal("bucket", &object, None, 3, 4).await.is_none());
let key = ReadRepairHealCacheKey::new("bucket", &object, None, 3, 4);
release_read_repair_heal_reservation(&key).await;
}
use crate::disk::error_reduce::is_ignored_err;
#[test]
fn cleanup_receipt_all_deletes_succeed_marks_full_reclaim() {
let errs = vec![None, None, None];
let attempted = vec![true, true, true];
let r = classify_old_data_dir_cleanup(&errs, &attempted, 2);
assert_eq!(r.attempted, 3);
assert_eq!(r.reclaimed, 3);
assert!(r.unreclaimed_disks.is_empty());
assert!(!r.below_quorum);
assert!(!r.has_residue());
}
mod reduce_common_old_current_size {
use super::*;
#[test]
fn agreement_at_quorum_wins() {
let observations = vec![
Some(OldCurrentSize::Present(5)),
Some(OldCurrentSize::Present(5)),
Some(OldCurrentSize::Present(5)),
Some(OldCurrentSize::Absent),
];
assert_eq!(
SetDisks::reduce_common_old_current_size(&observations, 3),
Some(OldCurrentSize::Present(5))
);
}
#[test]
fn absent_is_a_definite_vote() {
let observations = vec![
Some(OldCurrentSize::Absent),
Some(OldCurrentSize::Absent),
Some(OldCurrentSize::Absent),
None,
];
assert_eq!(SetDisks::reduce_common_old_current_size(&observations, 3), Some(OldCurrentSize::Absent));
}
#[test]
fn divergence_below_quorum_is_unknown() {
let observations = vec![
Some(OldCurrentSize::Present(5)),
Some(OldCurrentSize::Present(7)),
Some(OldCurrentSize::Absent),
Some(OldCurrentSize::Absent),
];
assert_eq!(SetDisks::reduce_common_old_current_size(&observations, 3), None);
}
#[test]
fn unknown_disks_do_not_vote() {
let observations = vec![Some(OldCurrentSize::Present(5)), Some(OldCurrentSize::Present(5)), None, None];
assert_eq!(SetDisks::reduce_common_old_current_size(&observations, 3), None);
}
#[test]
fn all_unknown_is_unknown() {
let observations: Vec<Option<OldCurrentSize>> = vec![None; 4];
assert_eq!(SetDisks::reduce_common_old_current_size(&observations, 3), None);
}
#[test]
fn unknown_majority_does_not_become_absent_quorum() {
let observations = vec![Some(OldCurrentSize::Present(5)), None, None, None];
assert_eq!(SetDisks::reduce_common_old_current_size(&observations, 3), None);
}
#[test]
fn empty_observations_are_unknown() {
assert_eq!(SetDisks::reduce_common_old_current_size(&[], 2), None);
}
}
#[test]
fn cleanup_receipt_not_found_counts_as_reclaimed() {
let errs = vec![None, Some(DiskError::FileNotFound), Some(DiskError::VolumeNotFound)];
let attempted = vec![true, true, true];
let r = classify_old_data_dir_cleanup(&errs, &attempted, 1);
assert_eq!(r.reclaimed, 3, "absent dir must count as already reclaimed");
assert!(r.unreclaimed_disks.is_empty());
}
#[test]
fn cleanup_receipt_transient_failures_are_residue_not_error() {
let errs = vec![
None,
None,
Some(DiskError::other("connection reset")),
Some(DiskError::other("io timeout")),
];
let attempted = vec![true, true, true, true];
let r = classify_old_data_dir_cleanup(&errs, &attempted, 3);
assert_eq!(r.attempted, 4);
assert_eq!(r.reclaimed, 2);
assert_eq!(r.unreclaimed_disks, vec![2, 3], "residue set must be exact");
assert!(r.below_quorum, "2 achieved < wq 3");
assert!(r.has_residue());
}
#[tokio::test]
async fn cleanup_join_error_maps_to_non_ignored_error_not_disk_not_found() {
let handle: tokio::task::JoinHandle<Option<DiskError>> = tokio::spawn(async { panic!("boom") });
let joined = handle.await;
assert!(joined.is_err(), "task must have panicked");
let mapped = map_cleanup_join_result(joined).expect("panic must surface as an error");
assert_ne!(
mapped,
DiskError::DiskNotFound,
"a task panic must never be masked as an ignorable DiskNotFound"
);
assert!(
!is_ignored_err(OBJECT_OP_IGNORED_ERRS, &mapped),
"a task panic must not be treated as an ignored error"
);
let r = classify_old_data_dir_cleanup(&[Some(mapped)], &[true], 1);
assert_eq!(r.unreclaimed_disks, vec![0]);
assert!(r.has_residue());
}
#[test]
fn cleanup_receipt_ignores_untargeted_none_slots() {
let errs = vec![Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
let attempted = vec![false, false];
let r = classify_old_data_dir_cleanup(&errs, &attempted, 1);
assert_eq!(r.attempted, 0);
assert_eq!(r.reclaimed, 0);
assert!(r.unreclaimed_disks.is_empty());
assert!(!r.has_residue(), "None-only slots never leak anything");
assert!(r.below_quorum, "parity lens still flags it, but nothing is actually leaked");
}
#[test]
fn cleanup_receipt_residue_and_below_quorum_are_independent_signals() {
let errs = vec![None, None, None, Some(DiskError::other("io"))];
let attempted = vec![true, true, true, true];
let r = classify_old_data_dir_cleanup(&errs, &attempted, 3);
assert!(!r.below_quorum, "3 achieved >= wq 3");
assert_eq!(r.unreclaimed_disks, vec![3]);
assert!(r.has_residue(), "leak metric must fire even when write quorum was met");
}
#[test]
fn persistent_cleanup_failure_triggers_leak_metric_and_heal_not_error() {
let errs = vec![None, Some(DiskError::other("disk down")), Some(DiskError::other("disk down"))];
let attempted = vec![true, true, true];
let receipt = classify_old_data_dir_cleanup(&errs, &attempted, 2);
assert!(receipt.has_residue());
let actions = old_data_dir_cleanup_actions(&receipt);
assert!(actions.emit_leak_metric, "persistent residue must be recorded by an observable metric");
assert!(actions.enqueue_heal, "persistent residue must enqueue a heal (disk-health signal)");
assert!(actions.warn);
}
#[test]
fn clean_cleanup_receipt_triggers_no_actions() {
let receipt = classify_old_data_dir_cleanup(&[None, None], &[true, true], 1);
assert_eq!(old_data_dir_cleanup_actions(&receipt), CleanupActions::default());
}
use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMeta, FileMetaVersion, MetaObject, RawFileInfo, VersionType};
use time::OffsetDateTime;
use uuid::Uuid;
fn raw_object_version(vid: Uuid, part_sizes: Vec<usize>) -> RawFileInfo {
let mut fm = FileMeta::new();
fm.add_version_filemata(FileMetaVersion {
version_type: VersionType::Object,
object: Some(MetaObject {
version_id: Some(vid),
erasure_algorithm: ErasureAlgo::ReedSolomon,
erasure_m: 2,
erasure_n: 1,
erasure_index: 1,
erasure_dist: vec![1, 2, 3],
erasure_block_size: 1 << 20,
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
part_numbers: vec![1, 2],
part_sizes,
part_actual_sizes: vec![10, 20],
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap()),
..Default::default()
}),
..Default::default()
})
.unwrap();
RawFileInfo {
buf: fm.marshal_msg().unwrap(),
}
}
#[tokio::test]
async fn pick_latest_quorum_masks_single_corrupt_disk_in_representative_slot() {
let vid = Uuid::new_v4();
let fileinfos = vec![
Some(raw_object_version(vid, vec![10])),
Some(raw_object_version(vid, vec![10, 20])),
Some(raw_object_version(vid, vec![10, 20])),
];
let errs = vec![None, None, None];
let (infos, out_errs) = SetDisks::pick_latest_quorum_files_info(fileinfos, errs, "bucket", "obj", false, false).await;
assert_eq!(out_errs[0], Some(DiskError::FileCorrupt), "corrupt representative disk must be flagged");
let good = infos.iter().filter(|fi| fi.is_valid()).count();
assert!(good >= 2, "quorum of good disks must survive corrupt representative, got {good}");
}
#[tokio::test]
async fn pick_latest_quorum_all_corrupt_fails_clean_without_panic() {
let vid = Uuid::new_v4();
let fileinfos = vec![
Some(raw_object_version(vid, vec![10])),
Some(raw_object_version(vid, vec![10])),
Some(raw_object_version(vid, vec![10])),
];
let errs = vec![None, None, None];
let (infos, out_errs) = SetDisks::pick_latest_quorum_files_info(fileinfos, errs, "bucket", "obj", false, false).await;
assert!(
out_errs.iter().all(|e| e == &Some(DiskError::FileCorrupt)),
"all disks must be flagged corrupt"
);
assert!(infos.iter().all(|fi| !fi.is_valid()), "no half-corrupt FileInfo may be returned");
}
}