use crate::diagnostics::get::{
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_SHARD_READ_ERROR_MISSING, GET_SHARD_READ_ERROR_NONE, GET_SHARD_READ_OUTCOME_ERROR,
GET_SHARD_READ_OUTCOME_MISSING, GET_SHARD_READ_OUTCOME_SUCCESS, GET_SHARD_ROLE_DATA, GET_SHARD_ROLE_PARITY, GET_STAGE_EMIT,
GET_STAGE_RANGE, GET_STAGE_RECONSTRUCT, GET_STAGE_STRIPE_READ, GET_STAGE_STRIPE_READ_FIRST_SHARD,
GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, get_stage_timer_if_enabled,
record_get_object_pipeline_failure, record_get_stage_duration_if_enabled,
};
use crate::disk::disk_store::get_object_disk_read_timeout;
use crate::disk::error::{Error, is_terminal_read_error};
use crate::disk::error_reduce::reduce_errs;
use crate::erasure::codec::workspace::ShardBufferPool;
use crate::erasure::coding::{BitrotReader, Erasure};
use crate::io_support::bitrot::DeferredReaderStripeHandle;
use crate::set_disk::shard_source::{
INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardReadRepair, ShardStripeSource, StripeReadState,
};
use futures::FutureExt;
use futures::stream::{FuturesUnordered, StreamExt};
use pin_project_lite::pin_project;
use smallvec::{SmallVec, smallvec};
use std::future::Future;
use std::io;
use std::io::ErrorKind;
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::io::AsyncWrite;
use tokio::io::AsyncWriteExt;
use tracing::{debug, error, warn};
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
type OwnedShardReadFuture<'a, R> =
Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
#[derive(Debug, Default)]
pub(crate) struct DecodeOutcome {
pub(crate) written: usize,
pub(crate) error: Option<std::io::Error>,
pub(crate) exact_quorum: bool,
pub(crate) repair: ShardReadRepair,
}
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
type StripeReadOutput = (ShardBuffers, ShardErrors);
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
const SHARD_LOCALITY_SCHEDULING_OFF: &str = "off";
const SHARD_LOCALITY_SCHEDULING_OBSERVE: &str = "observe";
const SHARD_LOCALITY_SCHEDULING_ON: &str = "on";
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) enum DecodeReadPolicy {
#[default]
Default,
DemandBound,
}
const MAX_DEMAND_BOUND_PARITY_IN_FLIGHT: usize = 4;
tokio::task_local! {
static DECODE_READ_POLICY: DecodeReadPolicy;
}
pub(crate) fn decode_read_policy() -> DecodeReadPolicy {
DECODE_READ_POLICY.try_with(|policy| *policy).unwrap_or_default()
}
pub(crate) async fn with_decode_read_policy<F>(policy: DecodeReadPolicy, future: F) -> F::Output
where
F: Future,
{
DECODE_READ_POLICY.scope(policy, future).await
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ShardLocalitySchedulingMode {
Off,
Observe,
On,
}
impl ShardLocalitySchedulingMode {
fn is_on(self) -> bool {
matches!(self, ShardLocalitySchedulingMode::On)
}
}
fn parse_shard_locality_scheduling_mode(value: &str) -> ShardLocalitySchedulingMode {
match value.trim() {
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OFF) => ShardLocalitySchedulingMode::Off,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OBSERVE) => ShardLocalitySchedulingMode::Observe,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_ON) => ShardLocalitySchedulingMode::On,
_ => ShardLocalitySchedulingMode::Off,
}
}
fn get_shard_locality_scheduling_mode() -> ShardLocalitySchedulingMode {
if let Some(value) = rustfs_utils::get_env_opt_str(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING) {
return parse_shard_locality_scheduling_mode(&value);
}
if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, false) {
return ShardLocalitySchedulingMode::On;
}
ShardLocalitySchedulingMode::Off
}
fn get_shard_locality_preference_enabled() -> bool {
get_shard_locality_scheduling_mode().is_on()
}
pub(crate) fn should_collect_shard_read_costs() -> bool {
rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_scheduling_mode().is_on()
}
const ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT: &str = "RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT";
const DEFAULT_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT: usize = 1;
fn get_decode_stripe_prefetch_count() -> usize {
rustfs_utils::get_env_usize(
ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT,
DEFAULT_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT,
)
}
const ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE: &str = "RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE";
const DEFAULT_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE: bool = false;
const ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: &str = "RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE";
const DEFAULT_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: bool = false;
pub(crate) fn get_lockstep_data_shards_only_enabled() -> bool {
matches!(decode_read_policy(), DecodeReadPolicy::DemandBound)
|| rustfs_utils::get_env_bool(
ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE,
DEFAULT_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE,
)
}
fn is_bitrot_decode_overlap_enabled() -> bool {
rustfs_utils::get_env_bool(
ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE,
DEFAULT_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE,
)
}
fn legacy_stripe_prefetch_enabled() -> bool {
!matches!(decode_read_policy(), DecodeReadPolicy::DemandBound)
&& (get_decode_stripe_prefetch_count() > 1 || is_bitrot_decode_overlap_enabled())
}
enum StripeFlow {
Continue,
Stop,
}
#[derive(Default)]
struct ShardReadCostCounts {
local: usize,
same_node: usize,
remote: usize,
unknown: usize,
}
impl ShardReadCostCounts {
fn record(&mut self, cost: ShardReadCost) {
match cost {
ShardReadCost::Local => self.local += 1,
ShardReadCost::SameNode => self.same_node += 1,
ShardReadCost::Remote => self.remote += 1,
ShardReadCost::Unknown => self.unknown += 1,
}
}
fn low_cost(&self) -> usize {
self.local + self.same_node
}
}
fn shard_read_launch_rank(cost: ShardReadCost) -> u8 {
match cost {
ShardReadCost::Local => 0,
ShardReadCost::SameNode => 1,
ShardReadCost::Remote => 2,
ShardReadCost::Unknown => 3,
}
}
#[allow(dead_code, reason = "launch ordering asserted by this file's tests (backlog#1823)")]
fn shard_read_launch_order(read_costs: &[ShardReadCost], num_readers: usize, locality_preference_enabled: bool) -> Vec<usize> {
let mut order: Vec<usize> = (0..num_readers).collect();
if locality_preference_enabled {
order.sort_by_key(|index| {
(
shard_read_launch_rank(read_costs.get(*index).copied().unwrap_or(ShardReadCost::Unknown)),
*index,
)
});
}
order
}
enum ReaderLaunchIter<'a, R> {
Original(std::iter::Enumerate<std::slice::IterMut<'a, Option<BitrotReader<R>>>>),
Locality(std::vec::IntoIter<(usize, &'a mut Option<BitrotReader<R>>)>),
}
impl<'a, R> ReaderLaunchIter<'a, R> {
fn new(readers: &'a mut [Option<BitrotReader<R>>], read_costs: &[ShardReadCost], locality_preference_enabled: bool) -> Self {
if !locality_preference_enabled {
return Self::Original(readers.iter_mut().enumerate());
}
let mut ordered_readers: Vec<_> = readers.iter_mut().enumerate().collect();
ordered_readers.sort_by_key(|(index, _)| {
(
shard_read_launch_rank(read_costs.get(*index).copied().unwrap_or(ShardReadCost::Unknown)),
*index,
)
});
Self::Locality(ordered_readers.into_iter())
}
}
impl<'a, R> Iterator for ReaderLaunchIter<'a, R> {
type Item = (usize, &'a mut Option<BitrotReader<R>>);
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Original(iter) => iter.next(),
Self::Locality(iter) => iter.next(),
}
}
}
fn shard_role(index: usize, data_shards: usize) -> &'static str {
if index < data_shards {
GET_SHARD_ROLE_DATA
} else {
GET_SHARD_ROLE_PARITY
}
}
#[allow(clippy::too_many_arguments)]
async fn read_shard_result<R>(
index: usize,
read_cost: ShardReadCost,
reader: &mut BitrotReader<R>,
recycled_buf: Option<Vec<u8>>,
shard_size: usize,
data_shards: usize,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> (Result<Vec<u8>, Error>, bool)
where
R: crate::erasure::coding::ShardSource,
{
let role = shard_role(index, data_shards);
let mut buf = recycled_buf.unwrap_or_else(|| Vec::with_capacity(shard_size));
buf.clear();
let read_start = metrics_path.map(|_| Instant::now());
let read_result = if read_timeout.is_zero() {
reader.read_appending(&mut buf, shard_size).await
} else {
match tokio::time::timeout(read_timeout, reader.read_appending(&mut buf, shard_size)).await {
Ok(result) => result,
Err(_) => {
let timeout_error = io::Error::new(ErrorKind::TimedOut, "shard read timed out");
let error_class = classify_io_error(&timeout_error).as_str();
if let Some(path) = metrics_path {
rustfs_io_metrics::record_get_object_shard_read_observation(
path,
index,
role,
read_cost.as_str(),
GET_SHARD_READ_OUTCOME_ERROR,
error_class,
0,
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
reader.last_verify_duration().as_secs_f64(),
);
}
return (Err(Error::from(timeout_error)), true);
}
}
};
match read_result {
Ok(n) => {
debug_assert_eq!(buf.len(), n, "read_appending must grow the buffer by exactly n");
if let Some(path) = metrics_path {
rustfs_io_metrics::record_get_object_shard_read_observation(
path,
index,
role,
read_cost.as_str(),
GET_SHARD_READ_OUTCOME_SUCCESS,
GET_SHARD_READ_ERROR_NONE,
n,
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
reader.last_verify_duration().as_secs_f64(),
);
}
(Ok(buf), false)
}
Err(e) => {
let verify_duration_secs = reader.last_verify_duration().as_secs_f64();
let error_class = classify_io_error(&e).as_str();
let should_retire = e.kind() == ErrorKind::TimedOut || is_terminal_read_error(&e);
if let Some(path) = metrics_path {
rustfs_io_metrics::record_get_object_shard_read_observation(
path,
index,
role,
read_cost.as_str(),
GET_SHARD_READ_OUTCOME_ERROR,
error_class,
0,
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
verify_duration_secs,
);
}
(Err(Error::from(e)), should_retire)
}
}
}
#[allow(clippy::too_many_arguments)]
fn read_shard<'a, R>(
index: usize,
read_cost: ShardReadCost,
reader: &'a mut Option<BitrotReader<R>>,
recycled_buf: Option<Vec<u8>>,
shard_size: usize,
data_shards: usize,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> ShardReadFuture<'a>
where
R: crate::erasure::coding::ShardSource + 'a,
{
let role = shard_role(index, data_shards);
if let Some(reader) = reader {
Box::pin(async move {
let (result, should_retire) = read_shard_result(
index,
read_cost,
reader,
recycled_buf,
shard_size,
data_shards,
read_timeout,
metrics_path,
)
.await;
(index, read_cost, result, should_retire)
})
} else {
Box::pin(async move {
if let Some(path) = metrics_path {
rustfs_io_metrics::record_get_object_shard_read_observation(
path,
index,
role,
read_cost.as_str(),
GET_SHARD_READ_OUTCOME_MISSING,
GET_SHARD_READ_ERROR_MISSING,
0,
0.0,
0.0,
);
}
(index, read_cost, Err(Error::FileNotFound), false)
})
}
}
#[allow(clippy::too_many_arguments)]
fn read_shard_owned<'a, R>(
index: usize,
read_cost: ShardReadCost,
reader: Option<BitrotReader<R>>,
recycled_buf: Option<Vec<u8>>,
shard_size: usize,
data_shards: usize,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> OwnedShardReadFuture<'a, R>
where
R: crate::erasure::coding::ShardSource + 'a,
{
let role = shard_role(index, data_shards);
Box::pin(async move {
let Some(mut reader) = reader else {
if let Some(path) = metrics_path {
rustfs_io_metrics::record_get_object_shard_read_observation(
path,
index,
role,
read_cost.as_str(),
GET_SHARD_READ_OUTCOME_MISSING,
GET_SHARD_READ_ERROR_MISSING,
0,
0.0,
0.0,
);
}
return (index, read_cost, Err(Error::FileNotFound), None, false);
};
let (result, should_retire) = read_shard_result(
index,
read_cost,
&mut reader,
recycled_buf,
shard_size,
data_shards,
read_timeout,
metrics_path,
)
.await;
(index, read_cost, result, Some(reader), should_retire)
})
}
#[allow(clippy::too_many_arguments)]
fn launch_owned_shard<'a, R>(
sets: &mut FuturesUnordered<OwnedShardReadFuture<'a, R>>,
readers: &mut [Option<BitrotReader<R>>],
buffers: &mut ShardBufferPool,
active: &mut [bool],
scheduled: &mut usize,
index: usize,
shard_size: usize,
data_shards: usize,
read_cost: ShardReadCost,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> bool
where
R: crate::erasure::coding::ShardSource + 'a,
{
let Some(reader) = readers.get_mut(index).and_then(Option::take) else {
return false;
};
launch_owned_reader(
sets,
buffers,
active,
scheduled,
index,
reader,
shard_size,
data_shards,
read_cost,
read_timeout,
metrics_path,
)
}
#[allow(clippy::too_many_arguments)]
fn launch_owned_reader<'a, R>(
sets: &mut FuturesUnordered<OwnedShardReadFuture<'a, R>>,
buffers: &mut ShardBufferPool,
active: &mut [bool],
scheduled: &mut usize,
index: usize,
reader: BitrotReader<R>,
shard_size: usize,
data_shards: usize,
read_cost: ShardReadCost,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> bool
where
R: crate::erasure::coding::ShardSource + 'a,
{
let recycled_buf = Some(buffers.take(index, shard_size));
*scheduled += 1;
active[index] = true;
sets.push(read_shard_owned(
index,
read_cost,
Some(reader),
recycled_buf,
shard_size,
data_shards,
read_timeout,
metrics_path,
));
true
}
pin_project! {
pub(crate) struct ParallelReader<R> {
#[pin]
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
shard_size: usize,
shard_file_size: usize,
data_shards: usize,
total_shards: usize,
metrics_path: Option<&'static str>,
read_costs: Vec<ShardReadCost>,
read_timeout: Duration,
verify_reconstruction: bool,
locality_preference_enabled: bool,
demand_bound_lockstep: bool,
buffers: ShardBufferPool,
stripe_state: Option<Box<StripeReadState>>,
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
hedged_readers: ActiveReaders,
stripe_index: usize,
integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
repair_evidence: Option<Arc<ShardReadRepair>>,
}
}
impl<R> ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
#[allow(
dead_code,
reason = "ParallelReader constructor used only by this file's tests (backlog#1823)"
)]
pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
None,
get_object_disk_read_timeout(),
false,
)
}
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
pub fn new_with_metrics_path(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
) -> Self {
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
get_object_disk_read_timeout(),
false,
)
}
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
pub fn new_with_metrics_path_and_read_costs(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
read_costs: Vec<ShardReadCost>,
) -> Self {
Self::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
read_costs,
get_object_disk_read_timeout(),
false,
)
}
fn new_for_decode(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
) -> Self {
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
get_object_disk_read_timeout(),
true,
)
}
pub(crate) fn new_with_metrics_path_and_reconstruction_verification(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
) -> Self {
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
get_object_disk_read_timeout(),
true,
)
}
pub(crate) fn new_with_metrics_path_read_costs_and_reconstruction_verification(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
read_costs: Vec<ShardReadCost>,
) -> Self {
Self::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
read_costs,
get_object_disk_read_timeout(),
true,
)
}
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
fn new_with_read_timeout(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
read_timeout: Duration,
) -> Self {
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
None,
read_timeout,
false,
)
}
fn new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
read_timeout: Duration,
verify_reconstruction: bool,
) -> Self {
let read_costs = vec![ShardReadCost::Unknown; readers.len()];
Self::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
e,
offset,
total_length,
metrics_path,
read_costs,
read_timeout,
verify_reconstruction,
)
}
#[allow(clippy::too_many_arguments)]
fn new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers: Vec<Option<BitrotReader<R>>>,
e: Erasure,
offset: usize,
total_length: usize,
metrics_path: Option<&'static str>,
mut read_costs: Vec<ShardReadCost>,
read_timeout: Duration,
verify_reconstruction: bool,
) -> Self {
let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
let shard_size = e.shard_size();
let shard_file_size = e.shard_file_size(total_length as i64) as usize;
let offset = (offset / e.block_size) * shard_size;
read_costs.resize(readers.len(), ShardReadCost::Unknown);
read_costs.truncate(readers.len());
let demand_bound_lockstep = get_lockstep_data_shards_only_enabled();
let engaged: SmallVec<_> = (0..readers.len())
.map(|index| !demand_bound_lockstep || index < e.data_shards)
.collect();
let hedged_readers = smallvec![false; readers.len()];
ParallelReader {
integrity: readers.iter().flatten().find_map(BitrotReader::integrity_proof),
readers,
offset,
shard_size,
shard_file_size,
data_shards: e.data_shards,
total_shards: e.data_shards + e.parity_shards,
metrics_path,
read_costs,
read_timeout,
verify_reconstruction,
locality_preference_enabled: get_shard_locality_preference_enabled(),
demand_bound_lockstep,
buffers: ShardBufferPool::new(e.data_shards + e.parity_shards),
stripe_state: None,
engaged,
deferred_handles: Vec::new(),
deferred_reopeners: Vec::new(),
hedged_readers,
stripe_index: 0,
repair_evidence: None,
}
}
pub(crate) fn with_repair_evidence(mut self, evidence: Arc<ShardReadRepair>) -> Self {
self.repair_evidence = Some(evidence);
self
}
pub(crate) fn with_deferred_parity_handles(mut self, mut handles: Vec<Option<DeferredReaderStripeHandle>>) -> Self {
handles.resize_with(self.readers.len(), || None);
handles.truncate(self.readers.len());
self.deferred_handles = handles;
self
}
pub(crate) fn with_deferred_parity_reopeners(mut self, mut reopeners: Vec<Option<DeferredReaderReopener<R>>>) -> Self {
reopeners.resize_with(self.readers.len(), || None);
reopeners.truncate(self.readers.len());
self.deferred_reopeners = reopeners;
self
}
}
#[allow(clippy::too_many_arguments)]
fn record_shard_read_result(
shards: &mut [Option<Vec<u8>>],
errs: &mut [Option<Error>],
retire_readers: &mut ShardIndexes,
success: &mut usize,
successful_costs: &mut ShardReadCostCounts,
i: usize,
read_cost: ShardReadCost,
result: Result<Vec<u8>, Error>,
should_retire: bool,
) -> bool {
match result {
Ok(v) => {
shards[i] = Some(v);
successful_costs.record(read_cost);
*success += 1;
false
}
Err(e) => {
errs[i] = Some(e);
if should_retire {
retire_readers.push(i);
}
true
}
}
}
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
for (i, active) in active_readers.iter().enumerate() {
if !*active {
continue;
}
if errs[i].is_none() {
errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read abandoned after read quorum")));
}
retire_readers.push(i);
warn!(shard_index = i, "retiring in-flight shard reader after read quorum");
}
}
fn shard_read_hedge_delay(read_timeout: Duration) -> Option<Duration> {
if read_timeout.is_zero() {
None
} else {
Some(read_timeout.min(Duration::from_millis(100)))
}
}
fn demand_bound_parity_admission_limit(shards: &[Option<Vec<u8>>], active: &[bool], data_shards: usize) -> usize {
let missing_data = shards.iter().take(data_shards).filter(|shard| shard.is_none()).count();
if missing_data == 0 {
return 0;
}
let successes = shards.iter().filter(|shard| shard.is_some()).count();
let needed_for_verification = (data_shards + 1).saturating_sub(successes);
let desired = missing_data
.saturating_add(1)
.min(needed_for_verification)
.min(MAX_DEMAND_BOUND_PARITY_IN_FLIGHT);
let active_parity = active
.iter()
.enumerate()
.skip(data_shards)
.filter(|(_, is_active)| **is_active)
.count();
desired.saturating_sub(active_parity)
}
fn shard_locality_remote_avoid_potential(remote_scheduled: usize, low_cost_available: usize, data_shards: usize) -> usize {
let theoretical_remote_needed = data_shards.saturating_sub(low_cost_available);
remote_scheduled.saturating_sub(theoretical_remote_needed)
}
#[allow(clippy::too_many_arguments)]
fn record_scheduled_read_cost(
read_cost: ShardReadCost,
locality_preference_enabled: bool,
low_cost_available: usize,
data_shards: usize,
count_remote_as_fallback: bool,
local_preferred: &mut usize,
remote_scheduled: &mut usize,
fallback_to_remote: &mut usize,
) {
if read_cost.is_low_cost() {
if locality_preference_enabled {
*local_preferred += 1;
}
} else if read_cost.is_remote() {
*remote_scheduled += 1;
if locality_preference_enabled && (count_remote_as_fallback || low_cost_available < data_shards) {
*fallback_to_remote += 1;
}
}
}
impl<R> ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
#[hotpath::measure(impl_type = "ParallelReader")]
pub async fn read(&mut self) -> StripeReadOutput {
let mut state = StripeReadState::with_slot_count(self.readers.len(), self.data_shards);
self.read_into_state(&mut state).await;
state.into_parts()
}
async fn read_into_state(&mut self, state: &mut StripeReadState) {
if self.verify_reconstruction {
self.read_lockstep(state).await;
return;
}
let num_readers = self.readers.len();
state.reset(num_readers, self.data_shards);
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
self.shard_file_size - self.offset
} else {
self.shard_size
};
if shard_size == 0 {
return;
}
self.offset += shard_size;
let (shards, errs) = state.parts_mut();
let read_costs = self.read_costs.as_slice();
let locality_preference_enabled = self.locality_preference_enabled;
let low_cost_available = self
.readers
.iter()
.enumerate()
.filter(|(index, reader)| {
reader.is_some()
&& read_costs
.get(*index)
.copied()
.unwrap_or(ShardReadCost::Unknown)
.is_low_cost()
})
.count();
let remote_available = self
.readers
.iter()
.enumerate()
.filter(|(index, reader)| {
reader.is_some() && read_costs.get(*index).copied().unwrap_or(ShardReadCost::Unknown).is_remote()
})
.count();
let mut successful_costs = ShardReadCostCounts::default();
let mut local_preferred = 0usize;
let mut fallback_to_remote = 0usize;
let mut remote_scheduled = 0usize;
self.buffers.ensure_slots(num_readers);
let mut retire_readers = ShardIndexes::new();
if num_readers >= self.data_shards {
let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled);
let mut sets = FuturesUnordered::new();
let mut active_readers: ActiveReaders = smallvec![false; num_readers];
let stripe_read_start = self.metrics_path.map(|_| Instant::now());
let mut scheduled = 0usize;
for _ in 0..self.data_shards {
if let Some((i, reader)) = reader_iter.next() {
let has_reader = reader.is_some();
let recycled_buf = if has_reader {
Some(self.buffers.take(i, shard_size))
} else {
None
};
let read_cost = read_costs.get(i).copied().unwrap_or(ShardReadCost::Unknown);
record_scheduled_read_cost(
read_cost,
locality_preference_enabled,
low_cost_available,
self.data_shards,
false,
&mut local_preferred,
&mut remote_scheduled,
&mut fallback_to_remote,
);
scheduled += 1;
active_readers[i] = has_reader;
sets.push(read_shard(
i,
read_cost,
reader,
recycled_buf,
shard_size,
self.data_shards,
self.read_timeout,
self.metrics_path,
));
}
}
let mut success = 0;
let mut completed = 0usize;
let mut failed = 0usize;
let mut first_shard_recorded = false;
let mut pending = sets.len();
let mut scheduled_all = false;
loop {
let item = if !scheduled_all {
match shard_read_hedge_delay(self.read_timeout) {
Some(hedge_delay) => {
let hedge_sleep = tokio::time::sleep(hedge_delay);
tokio::pin!(hedge_sleep);
tokio::select! {
item = sets.next() => item,
_ = &mut hedge_sleep => {
if let Some((next_i, next_reader)) = reader_iter.next() {
let has_reader = next_reader.is_some();
let recycled_buf = if has_reader {
Some(self.buffers.take(next_i, shard_size))
} else {
None
};
let next_read_cost = read_costs.get(next_i).copied().unwrap_or(ShardReadCost::Unknown);
record_scheduled_read_cost(
next_read_cost,
locality_preference_enabled,
low_cost_available,
self.data_shards,
true,
&mut local_preferred,
&mut remote_scheduled,
&mut fallback_to_remote,
);
active_readers[next_i] = has_reader;
scheduled += 1;
pending += 1;
sets.push(read_shard(
next_i,
next_read_cost,
next_reader,
recycled_buf,
shard_size,
self.data_shards,
self.read_timeout,
self.metrics_path,
));
} else {
scheduled_all = true;
}
continue;
}
}
}
None => sets.next().await,
}
} else {
sets.next().await
};
let Some((i, read_cost, result, should_retire)) = item else {
break;
};
pending = pending.saturating_sub(1);
active_readers[i] = false;
completed += 1;
if !first_shard_recorded {
if let Some(path) = self.metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_FIRST_SHARD, stripe_read_start);
}
first_shard_recorded = true;
}
let result_is_err = record_shard_read_result(
shards,
errs,
&mut retire_readers,
&mut success,
&mut successful_costs,
i,
read_cost,
result,
should_retire,
);
if result_is_err {
failed += 1;
if let Some((next_i, next_reader)) = reader_iter.next() {
let has_reader = next_reader.is_some();
let recycled_buf = if has_reader {
Some(self.buffers.take(next_i, shard_size))
} else {
None
};
let next_read_cost = read_costs.get(next_i).copied().unwrap_or(ShardReadCost::Unknown);
record_scheduled_read_cost(
next_read_cost,
locality_preference_enabled,
low_cost_available,
self.data_shards,
true,
&mut local_preferred,
&mut remote_scheduled,
&mut fallback_to_remote,
);
scheduled += 1;
active_readers[next_i] = has_reader;
pending += 1;
sets.push(read_shard(
next_i,
next_read_cost,
next_reader,
recycled_buf,
shard_size,
self.data_shards,
self.read_timeout,
self.metrics_path,
));
} else {
scheduled_all = true;
}
}
if success >= self.data_shards {
break;
}
if success + pending < self.data_shards {
break;
}
}
if success >= self.data_shards {
while let Some(Some((i, read_cost, result, should_retire))) = sets.next().now_or_never() {
active_readers[i] = false;
completed += 1;
if record_shard_read_result(
shards,
errs,
&mut retire_readers,
&mut success,
&mut successful_costs,
i,
read_cost,
result,
should_retire,
) {
failed += 1;
}
}
retire_abandoned_readers(errs, &mut retire_readers, &active_readers);
}
if let Some(path) = self.metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_QUORUM, stripe_read_start);
rustfs_io_metrics::record_get_object_shard_read_fanout(path, scheduled, completed, success, failed);
if locality_preference_enabled {
let remote_avoided = remote_available.saturating_sub(remote_scheduled);
rustfs_io_metrics::record_get_object_shard_locality_policy(
path,
local_preferred,
remote_avoided,
fallback_to_remote,
);
} else {
let remote_avoid_potential =
shard_locality_remote_avoid_potential(remote_scheduled, low_cost_available, self.data_shards);
rustfs_io_metrics::record_get_object_shard_locality_observe_only(
path,
remote_scheduled,
remote_avoid_potential,
);
rustfs_io_metrics::record_get_object_shard_locality_policy_disabled(path);
}
}
}
if let Some(path) = self.metrics_path {
rustfs_io_metrics::record_get_object_shard_read_cost_summary(
path,
successful_costs.local,
successful_costs.same_node,
successful_costs.remote,
successful_costs.unknown,
low_cost_available,
successful_costs.low_cost(),
self.data_shards,
low_cost_available >= self.data_shards,
);
}
for i in retire_readers {
self.readers[i] = None;
}
}
async fn read_lockstep(&mut self, state: &mut StripeReadState) {
if self.demand_bound_lockstep {
self.read_lockstep_demand_bound(state).await;
return;
}
let num_readers = self.readers.len();
state.reset(num_readers, self.data_shards);
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
self.shard_file_size - self.offset
} else {
self.shard_size
};
let (shards, errs) = state.parts_mut();
if shard_size == 0 {
return;
}
self.offset += shard_size;
let stripe_index = self.stripe_index;
self.stripe_index += 1;
self.buffers.ensure_slots(num_readers);
let missing_data_readers = self.readers.iter().take(self.data_shards).filter(|r| r.is_none()).count();
if missing_data_readers > 0 {
let want = missing_data_readers + 1;
let mut have = (self.data_shards..num_readers)
.filter(|&i| self.engaged[i] && self.readers[i].is_some())
.count();
for idx in self.data_shards..num_readers {
if have >= want {
break;
}
if self.readers[idx].is_some() && !self.engaged[idx] && self.try_engage_parity(idx, stripe_index) {
have += 1;
}
}
}
let mut bufs: ShardBuffers = SmallVec::with_capacity(num_readers);
for i in 0..num_readers {
bufs.push(if self.engaged[i] && self.readers[i].is_some() {
Some(self.buffers.take(i, shard_size))
} else {
None
});
}
let data_shards = self.data_shards;
let read_timeout = self.read_timeout;
let metrics_path = self.metrics_path;
let locality_preference_enabled = self.locality_preference_enabled;
let stripe_read_start = metrics_path.map(|_| Instant::now());
let mut retire_readers = ShardIndexes::new();
let mut scheduled = 0usize;
let mut success = 0usize;
let mut completed = 0usize;
let mut failed = 0usize;
let mut first_shard_recorded = false;
let mut hedged = false;
{
let mut sets = FuturesUnordered::new();
let reader_iter = ReaderLaunchIter::new(&mut self.readers, self.read_costs.as_slice(), locality_preference_enabled);
for (i, reader) in reader_iter {
if reader.is_none() {
continue;
}
let Some(recycled_buf) = bufs[i].take() else {
continue;
};
let read_cost = self.read_costs.get(i).copied().unwrap_or(ShardReadCost::Unknown);
scheduled += 1;
sets.push(read_shard(
i,
read_cost,
reader,
Some(recycled_buf),
shard_size,
data_shards,
read_timeout,
metrics_path,
));
}
let hedge_delay = shard_read_hedge_delay(read_timeout);
let hedge_sleep = hedge_delay.map(tokio::time::sleep);
tokio::pin!(hedge_sleep);
loop {
let item = match hedge_sleep.as_mut().as_pin_mut() {
Some(sleep) if !hedged => {
tokio::select! {
biased;
item = sets.next() => item,
_ = sleep => {
hedged = true;
let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if !data_missing || succeeded > data_shards {
break;
}
continue;
}
}
}
_ => sets.next().await,
};
let Some((i, _read_cost, result, _should_retire)) = item else {
break;
};
completed += 1;
if !first_shard_recorded {
if let Some(path) = metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_FIRST_SHARD, stripe_read_start);
}
first_shard_recorded = true;
}
match result {
Ok(v) => {
shards[i] = Some(v);
success += 1;
}
Err(e) => {
errs[i] = Some(e);
retire_readers.push(i);
failed += 1;
}
}
if hedged {
let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if !data_missing || succeeded > data_shards {
break;
}
}
}
}
if hedged {
for i in 0..num_readers {
if self.engaged[i] && self.readers[i].is_some() && shards[i].is_none() && errs[i].is_none() {
errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read hedged after a slow shard")));
retire_readers.push(i);
self.hedged_readers[i] = self.deferred_reopeners.get(i).is_some_and(Option::is_some);
}
}
}
for i in retire_readers.drain(..) {
self.readers[i] = None;
}
loop {
let success_now = shards.iter().filter(|shard| shard.is_some()).count();
let data_shard_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if !data_shard_missing || success_now > data_shards {
break;
}
let deferred_parity = (data_shards..num_readers).find(|&i| self.readers[i].is_some() && !self.engaged[i]);
let Some(idx) = deferred_parity.or_else(|| self.hedged_readers.iter().position(|hedged| *hedged)) else {
break;
};
if self.hedged_readers[idx] {
self.hedged_readers[idx] = false;
self.readers[idx] = self.deferred_reopeners[idx].as_ref().and_then(|reopen| reopen(stripe_index));
if self.readers[idx].is_none() {
continue;
}
} else if !self.try_engage_parity(idx, stripe_index) {
continue;
}
let read_cost = self.read_costs.get(idx).copied().unwrap_or(ShardReadCost::Unknown);
let recycled_buf = Some(self.buffers.take(idx, shard_size));
scheduled += 1;
let (i, _read_cost, result, _should_retire) = read_shard(
idx,
read_cost,
&mut self.readers[idx],
recycled_buf,
shard_size,
data_shards,
read_timeout,
metrics_path,
)
.await;
completed += 1;
match result {
Ok(v) => {
shards[i] = Some(v);
success += 1;
}
Err(e) => {
errs[i] = Some(e);
retire_readers.push(i);
failed += 1;
}
}
}
if let Some(path) = metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_QUORUM, stripe_read_start);
rustfs_io_metrics::record_get_object_shard_read_fanout(path, scheduled, completed, success, failed);
}
for i in retire_readers {
self.readers[i] = None;
}
}
async fn read_lockstep_demand_bound(&mut self, state: &mut StripeReadState) {
let num_readers = self.readers.len();
state.reset(num_readers, self.data_shards);
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
self.shard_file_size - self.offset
} else {
self.shard_size
};
let (shards, errs) = state.parts_mut();
if shard_size == 0 {
return;
}
self.offset += shard_size;
let stripe_index = self.stripe_index;
self.stripe_index += 1;
self.buffers.ensure_slots(num_readers);
let missing_data_readers = self.readers.iter().take(self.data_shards).filter(|r| r.is_none()).count();
let data_shards = self.data_shards;
let read_timeout = self.read_timeout;
let metrics_path = self.metrics_path;
let stripe_read_start = metrics_path.map(|_| Instant::now());
let mut retire_readers = ShardIndexes::new();
let mut scheduled = 0usize;
let mut success = 0usize;
let mut completed = 0usize;
let mut failed = 0usize;
let mut first_shard_recorded = false;
let mut active: ActiveReaders = smallvec![false; num_readers];
let mut temporary_parity: ActiveReaders = smallvec![false; num_readers];
let mut attempted_parity: ActiveReaders = smallvec![false; num_readers];
let mut data_failure_seen = missing_data_readers > 0;
let mut sets: FuturesUnordered<OwnedShardReadFuture<'_, R>> = FuturesUnordered::new();
let mut fallback_admitted = false;
for i in 0..num_readers {
if !self.engaged[i] || self.readers[i].is_none() {
continue;
}
let read_cost = self.read_costs.get(i).copied().unwrap_or(ShardReadCost::Unknown);
let _ = launch_owned_shard(
&mut sets,
&mut self.readers,
&mut self.buffers,
&mut active,
&mut scheduled,
i,
shard_size,
data_shards,
read_cost,
read_timeout,
metrics_path,
);
}
if missing_data_readers > 0 {
let want = (missing_data_readers + 1).min(MAX_DEMAND_BOUND_PARITY_IN_FLIGHT);
if self.launch_demand_bound_parity(
stripe_index,
want,
&mut sets,
&mut active,
&mut temporary_parity,
&mut attempted_parity,
false,
&mut scheduled,
shard_size,
data_shards,
read_timeout,
metrics_path,
) > 0
{
fallback_admitted = true;
}
}
let hedge_delay = shard_read_hedge_delay(read_timeout);
let hedge_sleep = hedge_delay.map(tokio::time::sleep);
tokio::pin!(hedge_sleep);
let mut hedged = false;
loop {
let item = if !hedged {
match hedge_sleep.as_mut().as_pin_mut() {
Some(sleep) => tokio::select! {
biased;
item = sets.next() => item,
_ = sleep => {
hedged = true;
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if data_missing {
let launched = self.launch_demand_bound_parity(
stripe_index,
demand_bound_parity_admission_limit(shards, &active, data_shards),
&mut sets,
&mut active,
&mut temporary_parity,
&mut attempted_parity,
true,
&mut scheduled,
shard_size,
data_shards,
read_timeout,
metrics_path,
);
if launched > 0 {
fallback_admitted = true;
}
}
continue;
}
},
None => sets.next().await,
}
} else {
sets.next().await
};
let Some((i, _read_cost, result, reader, should_retire)) = item else {
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
data_failure_seen |= errs.iter().take(data_shards).any(Option::is_some);
if data_missing && success <= data_shards {
let launched = self.launch_demand_bound_parity(
stripe_index,
demand_bound_parity_admission_limit(shards, &active, data_shards),
&mut sets,
&mut active,
&mut temporary_parity,
&mut attempted_parity,
!data_failure_seen,
&mut scheduled,
shard_size,
data_shards,
read_timeout,
metrics_path,
);
if launched > 0 {
fallback_admitted = true;
continue;
}
}
break;
};
let result_failed = result.is_err();
active[i] = false;
completed += 1;
if !first_shard_recorded {
if let Some(path) = metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_FIRST_SHARD, stripe_read_start);
}
first_shard_recorded = true;
}
match result {
Ok(v) => {
shards[i] = Some(v);
success += 1;
if temporary_parity[i] {
drop(reader);
self.engaged[i] = false;
} else if !should_retire {
self.readers[i] = reader;
}
}
Err(e) => {
failed += 1;
if i < data_shards {
data_failure_seen = true;
}
if temporary_parity[i] {
self.engaged[i] = false;
} else {
errs[i] = Some(e);
self.readers[i] = None;
retire_readers.push(i);
}
}
}
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if data_missing && success <= data_shards && (result_failed || fallback_admitted) {
let launched = self.launch_demand_bound_parity(
stripe_index,
demand_bound_parity_admission_limit(shards, &active, data_shards),
&mut sets,
&mut active,
&mut temporary_parity,
&mut attempted_parity,
!data_failure_seen,
&mut scheduled,
shard_size,
data_shards,
read_timeout,
metrics_path,
);
if launched > 0 {
fallback_admitted = true;
}
}
let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
if !data_missing {
break;
}
if fallback_admitted && succeeded > data_shards {
break;
}
}
for i in 0..num_readers {
if active[i] {
if temporary_parity[i] {
self.engaged[i] = false;
continue;
}
if shards[i].is_none() && errs[i].is_none() {
errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read hedged after a slow shard")));
retire_readers.push(i);
}
self.readers[i] = None;
}
}
drop(sets);
if let Some(path) = metrics_path {
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_QUORUM, stripe_read_start);
rustfs_io_metrics::record_get_object_shard_read_fanout(path, scheduled, completed, success, failed);
}
for i in retire_readers {
self.readers[i] = None;
}
}
#[allow(clippy::too_many_arguments)]
fn launch_demand_bound_parity<'a>(
&mut self,
stripe_index: usize,
max_new: usize,
sets: &mut FuturesUnordered<OwnedShardReadFuture<'a, R>>,
active: &mut [bool],
temporary_parity: &mut [bool],
attempted_parity: &mut [bool],
disposable: bool,
scheduled: &mut usize,
shard_size: usize,
data_shards: usize,
read_timeout: Duration,
metrics_path: Option<&'static str>,
) -> usize
where
R: 'a,
{
let mut launched = 0;
for idx in self.data_shards..self.readers.len() {
if launched >= max_new || self.engaged[idx] || self.readers[idx].is_none() || active[idx] || attempted_parity[idx] {
continue;
}
attempted_parity[idx] = true;
let reopener = self.deferred_reopeners.get(idx).and_then(Option::clone);
let (reader, temporary) = if disposable {
if let Some(reopener) = reopener {
let Some(reader) = reopener(stripe_index) else {
self.readers[idx] = None;
continue;
};
(reader, true)
} else {
if !self.try_engage_parity(idx, stripe_index) {
continue;
}
let Some(reader) = self.readers[idx].take() else {
self.engaged[idx] = false;
continue;
};
(reader, false)
}
} else if self.try_engage_parity(idx, stripe_index) {
let Some(reader) = self.readers[idx].take() else {
self.engaged[idx] = false;
continue;
};
(reader, false)
} else if let Some(reopener) = reopener {
let Some(reader) = reopener(stripe_index) else {
self.readers[idx] = None;
continue;
};
self.engaged[idx] = true;
(reader, false)
} else {
continue;
};
let read_cost = self.read_costs.get(idx).copied().unwrap_or(ShardReadCost::Unknown);
if launch_owned_reader(
sets,
&mut self.buffers,
active,
scheduled,
idx,
reader,
shard_size,
data_shards,
read_cost,
read_timeout,
metrics_path,
) {
temporary_parity[idx] = temporary;
launched += 1;
} else if !temporary {
self.engaged[idx] = false;
}
}
launched
}
fn try_engage_parity(&mut self, idx: usize, stripe_index: usize) -> bool {
if stripe_index == 0 {
self.engaged[idx] = true;
return true;
}
let advanced = self
.deferred_handles
.get(idx)
.and_then(|handle| handle.as_ref())
.is_some_and(|handle| handle.advance_stripes(stripe_index));
if advanced {
self.engaged[idx] = true;
true
} else {
warn!(
shard_index = idx,
stripe_index, "retiring parity reader that cannot be aligned to the current stripe"
);
self.readers[idx] = None;
false
}
}
pub fn recycle_shards(&mut self, shards: &mut [Option<Vec<u8>>]) {
for (i, reader) in self.readers.iter().enumerate() {
if reader.is_some()
&& let Some(buf) = shards.get_mut(i).and_then(Option::take)
{
self.buffers.put(i, buf);
}
}
}
}
#[async_trait::async_trait]
impl<R> ShardStripeSource for ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
async fn read_next_stripe(&mut self) -> Box<StripeReadState> {
let stripe = self.offset.checked_div(self.shard_size);
let mut state = self
.stripe_state
.take()
.unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
self.read_into_state(&mut state).await;
if let Some(evidence) = &self.repair_evidence {
evidence.observe(state.parts_mut().1);
}
if state.can_decode()
&& let Some(proof) = &self.integrity
{
let result = match stripe {
Some(stripe) => proof.reconstruction_proof(stripe, state.shards_mut()).await,
None => Err(io::Error::new(ErrorKind::InvalidData, "invalid integrity stripe offset")),
};
match result {
Ok(proof) => state.integrity = proof,
Err(_) => {
let (shards, errors) = state.parts_mut();
shards.iter_mut().for_each(|shard| *shard = None);
errors.fill(Some(Error::FileCorrupt));
}
}
}
state
}
fn recycle_stripe(&mut self, mut state: Box<StripeReadState>) {
self.recycle_shards(state.shards_mut());
state.reset(0, self.data_shards);
debug_assert!(self.stripe_state.is_none(), "a stripe cannot be recycled twice");
self.stripe_state = Some(state);
}
}
fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
let mut size = 0;
for shard in shards.iter().take(data_blocks).flatten() {
size += shard.len();
}
size
}
async fn read_stripe_timed<R>(reader: &mut ParallelReader<R>, stage_metrics_enabled: bool) -> StripeReadOutput
where
R: crate::erasure::coding::ShardSource,
{
let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let out = reader.read().await;
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, stripe_read_stage_start);
out
}
async fn write_data_blocks<W>(
writer: &mut W,
en_blocks: &[Option<Vec<u8>>],
data_blocks: usize,
mut offset: usize,
length: usize,
) -> std::io::Result<usize>
where
W: tokio::io::AsyncWrite + Send + Sync + Unpin,
{
if en_blocks.len() < data_blocks {
let reason = GetObjectFailureReason::RangeOrLengthInvalid;
record_get_object_pipeline_failure(GET_STAGE_RANGE, reason);
error!(
data_blocks,
available_shards = en_blocks.len(),
stage = GET_STAGE_RANGE,
reason = reason.as_str(),
"Write data blocks received fewer shards than data blocks"
);
return Err(io::Error::new(ErrorKind::InvalidInput, "data block count exceeds available shards"));
}
if length == 0 {
return Ok(0);
}
let Some(required_len) = offset.checked_add(length) else {
let reason = GetObjectFailureReason::RangeOrLengthInvalid;
record_get_object_pipeline_failure(GET_STAGE_RANGE, reason);
error!(
offset,
length,
stage = GET_STAGE_RANGE,
reason = reason.as_str(),
"Write data blocks offset and length overflow"
);
return Err(io::Error::new(ErrorKind::InvalidInput, "offset + length overflows"));
};
let data_len = get_data_block_len(en_blocks, data_blocks);
if data_len < required_len {
let reason = GetObjectFailureReason::ShortRead;
error!(
data_blocks,
data_len,
required_len,
offset,
length,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
"Write data blocks had insufficient data after offset"
);
record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
}
let mut total_written = 0;
let mut write_left = length;
for block_op in &en_blocks[..data_blocks] {
let Some(block) = block_op else {
let reason = GetObjectFailureReason::ShortRead;
error!(
data_blocks,
offset,
length,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
"Write data blocks found a missing data shard"
);
record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
};
if offset >= block.len() {
offset -= block.len();
continue;
}
let block_slice = &block[offset..];
offset = 0;
let write_len = write_left.min(block_slice.len());
let write_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
if let Err(e) = writer.write_all(&block_slice[..write_len]).await {
if let Some(write_stage_start) = write_stage_start {
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
}
let reason = classify_io_error(&e);
if reason == GetObjectFailureReason::DownstreamClosed {
debug!(
write_len,
total_written,
write_left,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
error = ?e,
"Write data blocks stopped after downstream closed"
);
} else {
record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
error!(
write_len,
total_written,
write_left,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
error = ?e,
"Write data blocks failed to emit bytes"
);
}
return Err(e);
}
if let Some(write_stage_start) = write_stage_start {
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
}
total_written += write_len;
write_left -= write_len;
if write_left == 0 {
return Ok(total_written);
}
}
let reason = GetObjectFailureReason::ShortRead;
error!(
total_written,
write_left,
length,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
"Write data blocks exhausted data before completing requested length"
);
record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"))
}
impl Erasure {
#[hotpath::measure(impl_type = "Erasure")]
pub async fn decode<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
) -> (usize, Option<std::io::Error>)
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
let outcome = self
.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
.await;
(outcome.written, outcome.error)
}
#[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
pub(crate) async fn decode_with_read_costs<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
read_costs: Vec<ShardReadCost>,
) -> (usize, Option<std::io::Error>)
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
let outcome = self
.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
.await;
(outcome.written, outcome.error)
}
#[allow(
dead_code,
reason = "kept as the compatibility wrapper for existing decode callers and tests"
)]
#[allow(clippy::too_many_arguments)]
pub(crate) async fn decode_with_stripe_handles<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
read_costs: Option<Vec<ShardReadCost>>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
) -> (usize, Option<std::io::Error>)
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
self.decode_with_stripe_handles_and_reopeners(
writer,
readers,
offset,
length,
total_length,
read_costs,
deferred_handles,
Vec::new(),
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn decode_with_stripe_handles_and_reopeners<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
read_costs: Option<Vec<ShardReadCost>>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
) -> (usize, Option<std::io::Error>)
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
let outcome = self
.decode_inner(
writer,
readers,
offset,
length,
total_length,
read_costs,
deferred_handles,
deferred_reopeners,
)
.await;
(outcome.written, outcome.error)
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn decode_with_stripe_handles_and_reopeners_with_diagnostics<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
read_costs: Option<Vec<ShardReadCost>>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
) -> DecodeOutcome
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
self.decode_inner(
writer,
readers,
offset,
length,
total_length,
read_costs,
deferred_handles,
deferred_reopeners,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn emit_decoded_stripe<W>(
&self,
writer: &mut W,
shards: &mut [Option<Vec<u8>>],
errs: &[Option<Error>],
block_offset: usize,
block_length: usize,
written: &mut usize,
ret_err: &mut Option<std::io::Error>,
repair: &ShardReadRepair,
stage_metrics_enabled: bool,
require_surplus_source: bool,
integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
request_offset: usize,
) -> StripeFlow
where
W: AsyncWrite + Send + Sync + Unpin,
{
repair.observe(errs);
if ret_err.is_none()
&& let (_, Some(err)) = reduce_errs(errs, &[])
&& (err == Error::FileNotFound || err == Error::FileCorrupt)
{
*ret_err = Some(err.into());
}
let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
if available_shards < self.data_shards {
let reason = GetObjectFailureReason::ReadQuorum;
error!(
data_shards = self.data_shards,
total_shards = self.data_shards + self.parity_shards,
available_shards,
block_offset,
block_length,
stage = GET_STAGE_STRIPE_READ,
reason = reason.as_str(),
errors = ?errs,
"Erasure decode could not gather enough shards"
);
record_get_object_pipeline_failure(GET_STAGE_STRIPE_READ, reason);
*ret_err = Some(Error::ErasureReadQuorum.into());
return StripeFlow::Stop;
}
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let proof = if let Some(integrity) = integrity {
match integrity
.reconstruction_proof((request_offset + *written) / self.block_size, shards)
.await
{
Ok(proof) => proof,
Err(error) => {
*ret_err = Some(error);
return StripeFlow::Stop;
}
}
} else {
None
};
let decode_result = if require_surplus_source {
self.decode_data_with_reconstruction_verification_for_lockstep(shards)
} else {
self.decode_data_with_reconstruction_verification(shards)
};
if let Err(e) = decode_result {
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
let reason = GetObjectFailureReason::DecodeError;
error!(
data_shards = self.data_shards,
total_shards = self.data_shards + self.parity_shards,
block_offset,
block_length,
stage = GET_STAGE_RECONSTRUCT,
reason = reason.as_str(),
error = ?e,
"Erasure shard reconstruction failed"
);
record_get_object_pipeline_failure(GET_STAGE_RECONSTRUCT, reason);
*ret_err = Some(e);
return StripeFlow::Stop;
}
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
if let Some(proof) = proof
&& let Err(error) = proof.verify(shards)
{
*ret_err = Some(error);
return StripeFlow::Stop;
}
let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let n = match write_data_blocks(writer, shards, self.data_shards, block_offset, block_length).await {
Ok(n) => {
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
n
}
Err(e) => {
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
let reason = classify_io_error(&e);
if reason == GetObjectFailureReason::DownstreamClosed {
debug!(
block_offset,
block_length,
bytes_written = *written,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
error = ?e,
"Erasure decode stopped after downstream closed"
);
} else {
error!(
block_offset,
block_length,
bytes_written = *written,
stage = GET_STAGE_EMIT,
reason = reason.as_str(),
error = ?e,
"Erasure decode failed to emit reconstructed data"
);
}
*ret_err = Some(e);
return StripeFlow::Stop;
}
};
*written += n;
StripeFlow::Continue
}
#[allow(clippy::too_many_arguments)]
async fn decode_inner<W, R>(
&self,
writer: &mut W,
readers: Vec<Option<BitrotReader<R>>>,
offset: usize,
length: usize,
total_length: usize,
read_costs: Option<Vec<ShardReadCost>>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
) -> DecodeOutcome
where
W: AsyncWrite + Send + Sync + Unpin,
R: crate::erasure::coding::ShardSource,
{
if readers.len() != self.data_shards + self.parity_shards {
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
return DecodeOutcome {
error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")),
..Default::default()
};
}
if self.block_size == 0 || self.data_shards == 0 {
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
return DecodeOutcome {
error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
..Default::default()
};
}
let Some(end_offset) = offset.checked_add(length) else {
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
return DecodeOutcome {
error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
..Default::default()
};
};
if end_offset > total_length {
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
return DecodeOutcome {
error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
..Default::default()
};
}
let mut ret_err = None;
let repair = ShardReadRepair::default();
if length == 0 {
return DecodeOutcome::default();
}
let mut written = 0;
let mut reader = if let Some(read_costs) = read_costs {
ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
self.clone(),
offset,
total_length,
Some(GET_OBJECT_PATH_LEGACY_DUPLEX),
read_costs,
get_object_disk_read_timeout(),
true,
)
} else {
ParallelReader::new_for_decode(readers, self.clone(), offset, total_length, Some(GET_OBJECT_PATH_LEGACY_DUPLEX))
}
.with_deferred_parity_handles(deferred_handles)
.with_deferred_parity_reopeners(deferred_reopeners);
let integrity = reader.integrity.clone();
let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
let start = offset / self.block_size;
let end = end_offset.saturating_sub(1) / self.block_size;
let block_geometry = |i: usize| -> (usize, usize) {
if start == end {
(offset % self.block_size, length)
} else if i == start {
(offset % self.block_size, self.block_size - (offset % self.block_size))
} else if i == end {
let end_remainder = end_offset % self.block_size;
(0, if end_remainder == 0 { self.block_size } else { end_remainder })
} else {
(0, self.block_size)
}
};
let mut exact_quorum = false;
if legacy_stripe_prefetch_enabled() {
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let mut blocks = Vec::with_capacity(end - start + 1);
for i in start..=end {
let geometry = block_geometry(i);
if geometry.1 == 0 {
break;
}
blocks.push(geometry);
}
if !blocks.is_empty() {
let mut current = Some(read_stripe_timed(&mut reader, stage_metrics_enabled).await);
for idx in 0..blocks.len() {
let (block_offset, block_length) = blocks[idx];
let Some((mut shards, errs)) = current.take() else {
break;
};
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
if idx + 1 < blocks.len() {
let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
let emit_fut = self.emit_decoded_stripe(
writer,
&mut shards,
&errs,
block_offset,
block_length,
&mut written,
&mut ret_err,
&repair,
stage_metrics_enabled,
require_surplus_source,
integrity.as_ref(),
offset,
);
tokio::pin!(read_fut);
tokio::pin!(emit_fut);
let mut next: Option<StripeReadOutput> = None;
let mut flow: Option<StripeFlow> = None;
while flow.is_none() || (matches!(flow, Some(StripeFlow::Continue)) && next.is_none()) {
tokio::select! {
biased;
f = &mut emit_fut, if flow.is_none() => flow = Some(f),
n = &mut read_fut, if next.is_none() => next = Some(n),
}
}
(flow, next)
};
match flow {
Some(StripeFlow::Continue) => {
reader.recycle_shards(&mut shards);
current = next;
}
_ => break,
}
} else {
match self
.emit_decoded_stripe(
writer,
&mut shards,
&errs,
block_offset,
block_length,
&mut written,
&mut ret_err,
&repair,
stage_metrics_enabled,
require_surplus_source,
integrity.as_ref(),
offset,
)
.await
{
StripeFlow::Continue => reader.recycle_shards(&mut shards),
StripeFlow::Stop => break,
}
}
}
}
} else {
for i in start..=end {
let (block_offset, block_length) = block_geometry(i);
if block_length == 0 {
break;
}
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let (mut shards, errs) = reader.read().await;
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
record_get_stage_duration_if_enabled(
GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_STAGE_STRIPE_READ,
stripe_read_stage_start,
);
match self
.emit_decoded_stripe(
writer,
&mut shards,
&errs,
block_offset,
block_length,
&mut written,
&mut ret_err,
&repair,
stage_metrics_enabled,
require_surplus_source,
integrity.as_ref(),
offset,
)
.await
{
StripeFlow::Continue => reader.recycle_shards(&mut shards),
StripeFlow::Stop => break,
}
}
}
if ret_err.is_none() && written < length {
ret_err = Some(Error::LessData.into());
}
DecodeOutcome {
written,
error: ret_err,
exact_quorum,
repair,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
disk::error::DiskError,
erasure::coding::{BitrotReader, BitrotWriter},
};
use rustfs_utils::HashAlgorithm;
use std::future::Future;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
};
use std::task::{Context, Poll};
use tokio::io::AsyncRead;
use tokio::io::ReadBuf;
use tokio::time::{Instant as TokioInstant, Sleep};
type BoxedShardReader = crate::io_support::bitrot::ShardReader;
#[test]
fn parallel_reader_keeps_stripe_scratch_out_of_line() {
assert_eq!(
std::mem::size_of::<Option<Box<StripeReadState>>>(),
std::mem::size_of::<usize>(),
"the request-scoped cache must remain pointer-sized",
);
}
#[tokio::test]
async fn parallel_reader_preserves_slot_count_above_inline_capacity() {
const DATA_SHARDS: usize = INLINE_SHARD_SLOTS;
const TOTAL_SHARDS: usize = INLINE_SHARD_SLOTS + 1;
let readers = std::iter::repeat_with(|| None).take(TOTAL_SHARDS).collect();
let erasure = Erasure::new(DATA_SHARDS, 1, DATA_SHARDS);
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, DATA_SHARDS);
let (shards, errors) = reader.read().await;
assert!(shards.spilled());
assert!(errors.spilled());
assert_eq!(shards.len(), TOTAL_SHARDS);
assert_eq!(errors.len(), TOTAL_SHARDS);
}
#[tokio::test]
async fn codec_reader_reuses_inline_and_spilled_stripe_scratch_between_reads() {
for total_shards in [INLINE_SHARD_SLOTS, INLINE_SHARD_SLOTS + 1] {
let data_shards = total_shards - 1;
let readers = std::iter::repeat_with(|| None).take(total_shards).collect();
let erasure = Erasure::new(data_shards, 1, data_shards * 2);
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, data_shards * 2);
let first = ShardStripeSource::read_next_stripe(&mut reader).await;
let first_state = (&*first) as *const StripeReadState;
let first_storage = first.scratch_storage();
assert_eq!(first_storage.2, total_shards > INLINE_SHARD_SLOTS);
assert_eq!(first_storage.3, total_shards > INLINE_SHARD_SLOTS);
ShardStripeSource::recycle_stripe(&mut reader, first);
let second = ShardStripeSource::read_next_stripe(&mut reader).await;
let second_storage = second.scratch_storage();
assert_eq!(
(&*second) as *const StripeReadState,
first_state,
"the request-scoped state must be reused"
);
assert_eq!(second_storage.0, first_storage.0, "shard slots must reuse their allocation");
assert_eq!(second_storage.1, first_storage.1, "error slots must reuse their allocation");
assert_eq!(second.into_parts().0.len(), total_shards);
}
}
#[tokio::test]
async fn codec_reader_returns_shard_allocations_to_the_request_pool() {
const SHARD_SIZE: usize = 16;
let hash_algo = HashAlgorithm::None;
let readers = vec![Some(create_reader(SHARD_SIZE, 2, 0x5a, &hash_algo, false).await)];
let erasure = Erasure::new(1, 0, SHARD_SIZE);
let mut reader = ParallelReader::new(readers, erasure, 0, SHARD_SIZE * 2);
let first = ShardStripeSource::read_next_stripe(&mut reader).await;
let first_allocation = first
.shard_allocation(0)
.expect("the first stripe should own its shard allocation");
ShardStripeSource::recycle_stripe(&mut reader, first);
assert_eq!(
reader.buffers.stored_allocation(0),
Some(first_allocation),
"recycling a stripe must return its shard allocation to the request pool"
);
let second = ShardStripeSource::read_next_stripe(&mut reader).await;
assert_eq!(
second.shard_allocation(0),
Some(first_allocation),
"the next stripe must reuse the pooled shard allocation"
);
}
struct CountingShardReader {
inner: Cursor<Vec<u8>>,
bytes_read: Arc<AtomicUsize>,
}
impl crate::erasure::coding::ShardSource for CountingShardReader {}
impl AsyncRead for CountingShardReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let before = buf.filled().len();
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
if let Poll::Ready(Ok(())) = result {
let delta = buf.filled().len() - before;
self.bytes_read.fetch_add(delta, Ordering::SeqCst);
}
result
}
}
fn readers_with_deferred_parity(
shard_bufs: &[Vec<u8>],
data_shards: usize,
shard_size: usize,
hash_algo: &HashAlgorithm,
truncate: &[(usize, usize)],
) -> (Vec<Option<BitrotReader<BoxedShardReader>>>, Vec<Option<DeferredReaderStripeHandle>>) {
use crate::io_support::bitrot::create_deferred_bitrot_reader_with_stripe_handle;
let mut readers = Vec::with_capacity(shard_bufs.len());
let mut handles: Vec<Option<DeferredReaderStripeHandle>> = vec![None; shard_bufs.len()];
for (i, buf) in shard_bufs.iter().enumerate() {
if i < data_shards {
let bytes = truncate
.iter()
.find(|(index, _)| *index == i)
.map(|(_, len)| buf[..*len].to_vec())
.unwrap_or_else(|| buf.clone());
readers.push(Some(BitrotReader::new(
crate::io_support::bitrot::ShardReader::Stream(Box::new(Cursor::new(bytes))),
shard_size,
hash_algo.clone(),
false,
)));
} else {
let (reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
Some(bytes::Bytes::from(buf.clone())),
None,
"test-bucket",
"test-object",
0,
buf.len(),
shard_size,
hash_algo.clone(),
false,
false,
);
readers.push(Some(reader));
handles[i] = Some(handle);
}
}
(readers, handles)
}
enum TestShardReader {
Ready(Cursor<Vec<u8>>),
ReadyAt {
cursor: Cursor<Vec<u8>>,
ready_at: TokioInstant,
sleep: Option<Pin<Box<Sleep>>>,
},
Pending,
Parked,
PartialThenPending {
data: Vec<u8>,
emitted: bool,
},
TimedOut,
TerminalFileNotFound,
PrefixThenSlow {
cursor: Cursor<Vec<u8>>,
stall: Duration,
sleep: Option<Pin<Box<Sleep>>>,
stall_polls: Arc<AtomicUsize>,
},
}
impl crate::erasure::coding::ShardSource for TestShardReader {}
impl AsyncRead for TestShardReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
match &mut *self {
TestShardReader::Ready(cursor) => Pin::new(cursor).poll_read(cx, buf),
TestShardReader::ReadyAt { cursor, ready_at, sleep } => {
if TokioInstant::now() < *ready_at {
let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(*ready_at)));
if sleeper.as_mut().poll(cx).is_pending() {
return Poll::Pending;
}
}
Pin::new(cursor).poll_read(cx, buf)
}
TestShardReader::Pending => {
cx.waker().wake_by_ref();
Poll::Pending
}
TestShardReader::Parked => Poll::Pending,
TestShardReader::PartialThenPending { data, emitted } => {
if *emitted {
cx.waker().wake_by_ref();
return Poll::Pending;
}
let len = data.len().min(buf.remaining());
buf.put_slice(&data[..len]);
*emitted = true;
Poll::Ready(Ok(()))
}
TestShardReader::TimedOut => Poll::Ready(Err(io::Error::new(ErrorKind::TimedOut, "test shard read timed out"))),
TestShardReader::TerminalFileNotFound => {
Poll::Ready(Err(crate::disk::error::terminal_read_error_to_io(Error::FileNotFound)))
}
TestShardReader::PrefixThenSlow {
cursor,
stall,
sleep,
stall_polls,
} => {
let before = buf.filled().len();
match Pin::new(cursor).poll_read(cx, buf) {
Poll::Ready(Ok(())) if buf.filled().len() != before => Poll::Ready(Ok(())),
Poll::Ready(Ok(())) => {
stall_polls.fetch_add(1, Ordering::SeqCst);
let stall = *stall;
let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep(stall)));
let _ = sleeper.as_mut().poll(cx);
Poll::Pending
}
other => other,
}
}
}
}
}
struct FailingEmitWriter;
impl AsyncWrite for FailingEmitWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure")))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
struct YieldOnceThenFailWriter {
yielded: bool,
}
impl AsyncWrite for YieldOnceThenFailWriter {
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
if !self.yielded {
self.yielded = true;
cx.waker().wake_by_ref();
return Poll::Pending;
}
Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure after prefetch poll")))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
struct DownstreamClosedWriter;
impl AsyncWrite for DownstreamClosedWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
Poll::Ready(Err(crate::diagnostics::get::mark_get_object_downstream_closed(io::Error::new(
ErrorKind::BrokenPipe,
"injected downstream close",
))))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[derive(Clone, Default)]
struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
impl CapturedLogs {
fn contents(&self) -> String {
String::from_utf8(self.0.lock().expect("captured logs mutex should not be poisoned").clone())
.expect("captured logs should be valid UTF-8")
}
}
impl std::io::Write for CapturedLogWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.expect("captured logs mutex should not be poisoned")
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
type Writer = CapturedLogWriter;
fn make_writer(&'a self) -> Self::Writer {
CapturedLogWriter(Arc::clone(&self.0))
}
}
#[test]
fn parallel_reader_constructor_variants_preserve_read_cost_and_verification_flags() {
let erasure = Erasure::new(2, 1, 64);
let readers = vec![None, None, None];
let read_costs = vec![ShardReadCost::Local, ShardReadCost::Remote, ShardReadCost::Unknown];
let reader: ParallelReader<Cursor<Vec<u8>>> =
ParallelReader::new_with_metrics_path_read_costs_and_reconstruction_verification(
readers,
erasure.clone(),
64,
17,
Some("constructor-test"),
read_costs.clone(),
);
assert_eq!(reader.offset, reader.shard_size);
assert_eq!(reader.read_costs, read_costs);
assert!(reader.verify_reconstruction);
let defaulted: ParallelReader<Cursor<Vec<u8>>> =
ParallelReader::new_with_metrics_path_and_reconstruction_verification(vec![None, None, None], erasure, 0, 64, None);
assert_eq!(defaulted.read_costs, vec![ShardReadCost::Unknown; 3]);
assert!(defaulted.verify_reconstruction);
}
#[tokio::test]
async fn test_write_data_blocks_writes_range_across_blocks() {
let blocks = vec![Some(vec![1, 2, 3, 4]), Some(vec![5, 6, 7]), Some(vec![8, 9])];
let mut out = Vec::new();
let written = write_data_blocks(&mut out, &blocks, 3, 2, 5).await.unwrap();
assert_eq!(written, 5);
assert_eq!(out, vec![3, 4, 5, 6, 7]);
}
#[tokio::test]
async fn test_write_data_blocks_rejects_short_data_after_offset() {
let blocks = vec![Some(vec![1, 2, 3, 4]), Some(vec![5, 6, 7])];
let mut out = Vec::new();
let err = write_data_blocks(&mut out, &blocks, 2, 3, 5).await.unwrap_err();
assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
assert!(out.is_empty());
}
#[tokio::test]
async fn test_write_data_blocks_rejects_invalid_data_block_count() {
let blocks = vec![Some(vec![1, 2, 3, 4])];
let mut out = Vec::new();
let err = write_data_blocks(&mut out, &blocks, 2, 0, 1).await.unwrap_err();
assert_eq!(err.kind(), ErrorKind::InvalidInput);
assert!(out.is_empty());
}
#[tokio::test]
async fn test_write_data_blocks_rejects_offset_length_overflow() {
let blocks = vec![Some(vec![1, 2, 3, 4])];
let mut out = Vec::new();
let err = write_data_blocks(&mut out, &blocks, 1, usize::MAX, 1).await.unwrap_err();
assert_eq!(err.kind(), ErrorKind::InvalidInput);
assert!(out.is_empty());
}
#[tokio::test]
async fn test_write_data_blocks_rejects_missing_data_shard_even_when_total_bytes_are_available() {
let blocks = vec![None, Some(vec![1, 2, 3, 4])];
let mut out = Vec::new();
let err = write_data_blocks(&mut out, &blocks, 2, 0, 1).await.unwrap_err();
assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
assert!(out.is_empty());
}
#[tokio::test]
async fn test_write_data_blocks_propagates_writer_emit_failure() {
let blocks = vec![Some(vec![1, 2, 3, 4])];
let mut writer = FailingEmitWriter;
let err = write_data_blocks(&mut writer, &blocks, 1, 0, 4)
.await
.expect_err("writer failure must fail the decoded emit path");
assert_eq!(err.kind(), ErrorKind::BrokenPipe);
assert_eq!(err.to_string(), "injected emit failure");
}
#[tokio::test(flavor = "current_thread")]
async fn erasure_decode_logs_reconstructed_downstream_close_at_debug() {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG)
.with_writer(logs.clone())
.with_ansi(false)
.without_time()
.finish();
let _guard = tracing::subscriber::set_default(subscriber);
let erasure = Erasure::new(2, 1, 64);
let data: Vec<u8> = (0..64).collect();
let shard_size = erasure.shard_size();
let encoded = erasure.encode_data(&data).expect("test data should encode");
let readers = vec![
None,
Some(BitrotReader::new(
Cursor::new(encoded[1].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
Cursor::new(encoded[2].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
];
let mut writer = DownstreamClosedWriter;
let (written, err) = erasure.decode(&mut writer, readers, 0, data.len(), data.len()).await;
assert_eq!(written, 0);
assert_eq!(err.expect("downstream close must still terminate the GET").kind(), ErrorKind::BrokenPipe);
let captured = logs.contents();
assert!(captured.contains("Erasure decode stopped after downstream closed"));
assert!(!captured.contains("Erasure decode failed to emit reconstructed data"));
}
#[tokio::test]
async fn test_erasure_decode_rejects_reader_count_and_range_overflow() {
let erasure = Erasure::new(2, 1, 64);
let mut output = Vec::new();
let (written, err) = erasure
.decode(&mut output, Vec::<Option<BitrotReader<Cursor<Vec<u8>>>>>::new(), 0, 1, 1)
.await;
assert_eq!(written, 0);
assert_eq!(err.expect("reader count mismatch should fail").kind(), ErrorKind::InvalidInput);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
let (written, err) = erasure.decode(&mut output, readers, usize::MAX, 1, usize::MAX).await;
assert_eq!(written, 0);
assert_eq!(err.expect("offset overflow should fail").kind(), ErrorKind::InvalidInput);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
let (written, err) = erasure.decode(&mut output, readers, 2, 8, 9).await;
assert_eq!(written, 0);
assert_eq!(err.expect("range beyond total length should fail").kind(), ErrorKind::InvalidInput);
}
#[tokio::test]
async fn test_erasure_decode_zero_length_does_not_read_or_emit() {
let erasure = Erasure::new(2, 1, 64);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, 0, 0).await;
assert_eq!(written, 0);
assert!(err.is_none());
assert!(output.is_empty());
}
#[tokio::test]
async fn test_erasure_decode_with_read_costs_restores_missing_data_shard_range() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let data: Vec<u8> = (0..BLOCK_SIZE as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let encoded = erasure.encode_data(&data).expect("encode should succeed");
let readers = vec![
None,
Some(BitrotReader::new(
Cursor::new(encoded[1].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
Cursor::new(encoded[DATA_SHARDS].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
Cursor::new(encoded[DATA_SHARDS + 1].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
];
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Remote,
ShardReadCost::Unknown,
];
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 5, 37, data.len(), read_costs)
.await;
assert!(err.is_none(), "missing data shard should reconstruct with parity: {err:?}");
assert_eq!(written, 37);
assert_eq!(output, data[5..42]);
}
#[tokio::test]
async fn test_erasure_decode_ranged_read_returns_correct_bytes() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
while offset < total_len {
let end = (offset + BLOCK_SIZE).min(total_len);
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
offset = end;
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let hash_size = hash_algo.size();
let make_readers = |off: usize| -> Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> {
let start_block = off / BLOCK_SIZE;
let cursor_pos = start_block * (shard_size + hash_size);
shard_bufs
.iter()
.map(|buf| {
let mut cursor = Cursor::new(buf.clone());
cursor.set_position(cursor_pos as u64);
Some(BitrotReader::new(cursor, shard_size, hash_algo.clone(), false))
})
.collect()
};
let cases: &[(usize, usize, &str)] = &[
(0, total_len, "full read"),
(0, 50, "head from start, partial block"),
(10, 30, "small range within first block"),
(60, 80, "range crossing two block boundaries"),
(128, 50, "range starting at block boundary"),
(130, 10, "small range deep in middle"),
(192, 8, "tail covering last partial block"),
];
for &(off, len, desc) in cases {
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, make_readers(off), off, len, total_len).await;
assert!(err.is_none(), "{}: unexpected error: {:?}", desc, err);
assert_eq!(written, len, "{}: written != length", desc);
assert_eq!(output, total_data[off..off + len], "{}: bytes mismatch", desc);
}
}
async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec<Vec<u8>> {
let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256))
.collect();
for block in payload.chunks(erasure.block_size) {
let shards = erasure.encode_data(block).expect("copy source should encode");
for (index, shard) in shards.iter().enumerate() {
let mut bytes = shard.to_vec();
if corrupt_parity && index == erasure.data_shards {
bytes[0] ^= 0x80;
}
writers[index].write(&bytes).await.expect("copy source should frame");
}
}
writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect()
}
#[tokio::test]
#[serial_test::serial]
async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() {
const BLOCK_SIZE: usize = 768;
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
.map(|index| (index.wrapping_mul(31) % 251) as u8)
.collect();
for uses_legacy in [false, true] {
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] {
for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] {
let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size());
let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect();
let (mut readers, handles) =
readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
for index in offline {
readers[index] = None;
}
let mut output = Vec::new();
let (written, error) = crate::set_disk::with_get_object_read_policy(
crate::set_disk::GetObjectReadPolicy::CopySource,
erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles),
)
.await;
assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}");
assert_eq!(written, length);
assert_eq!(output, payload[offset..offset + length]);
}
}
}
}
#[tokio::test]
#[serial_test::serial]
async fn copy_source_recovers_exact_quorum_after_first_stripe() {
const BLOCK_SIZE: usize = 768;
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
.map(|index| (index.wrapping_mul(31) % 251) as u8)
.collect();
for uses_legacy in [false, true] {
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size();
let (readers, handles) = readers_with_deferred_parity(
&buffers,
12,
erasure.shard_size(),
&HashAlgorithm::HighwayHash256,
&[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)],
);
let mut output = Vec::new();
let (written, error) = crate::set_disk::with_get_object_read_policy(
crate::set_disk::GetObjectReadPolicy::CopySource,
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
)
.await;
assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}");
assert_eq!(written, payload.len());
assert_eq!(output, payload);
}
}
#[tokio::test]
#[serial_test::serial]
async fn copy_source_below_read_quorum_fails_before_output() {
const BLOCK_SIZE: usize = 768;
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
for uses_legacy in [false, true] {
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
let (mut readers, handles) =
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
for reader in readers.iter_mut().take(5) {
*reader = None;
}
let mut output = Vec::new();
let (written, error) = crate::set_disk::with_get_object_read_policy(
crate::set_disk::GetObjectReadPolicy::CopySource,
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
)
.await;
assert_eq!(written, 0);
assert!(output.is_empty());
assert_eq!(
error
.expect("eleven shards cannot reconstruct twelve data shards")
.to_string(),
Error::ErasureReadQuorum.to_string()
);
}
}
#[tokio::test]
#[serial_test::serial]
async fn copy_source_rejects_corrupt_surplus_parity_before_output() {
const BLOCK_SIZE: usize = 768;
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
for uses_legacy in [false, true] {
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await;
let (mut readers, handles) =
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
for reader in readers.iter_mut().take(3) {
*reader = None;
}
let mut output = Vec::new();
let (written, error) = crate::set_disk::with_get_object_read_policy(
crate::set_disk::GetObjectReadPolicy::CopySource,
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
)
.await;
assert_eq!(written, 0);
assert!(output.is_empty());
let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing");
assert_eq!(error.kind(), ErrorKind::InvalidData);
assert!(error.to_string().contains("inconsistent read source shards"));
}
}
#[tokio::test]
async fn test_erasure_decode_with_missing_shards_reuses_buffers() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
while offset < total_len {
let end = (offset + BLOCK_SIZE).min(total_len);
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
offset = end;
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let missing = [0usize, 2usize];
for verify in [true, false] {
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = shard_bufs
.iter()
.enumerate()
.map(|(i, buf)| {
if missing.contains(&i) {
None
} else {
Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), !verify))
}
})
.collect();
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
assert!(err.is_none(), "verify={verify}: unexpected error: {err:?}");
assert_eq!(written, total_len, "verify={verify}: short write");
assert_eq!(output, total_data, "verify={verify}: reconstructed bytes mismatch");
}
}
#[tokio::test]
async fn test_erasure_decode_rejects_inconsistent_reconstruction_sources() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let data: Vec<u8> = (0..BLOCK_SIZE as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let encoded = erasure.encode_data(&data).expect("encode should succeed");
let mut corrupt_parity = encoded[DATA_SHARDS].to_vec();
corrupt_parity[0] ^= 0x80;
let readers = vec![
None,
Some(BitrotReader::new(
Cursor::new(encoded[1].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(Cursor::new(corrupt_parity), shard_size, HashAlgorithm::None, false)),
Some(BitrotReader::new(
Cursor::new(encoded[DATA_SHARDS + 1].to_vec()),
shard_size,
HashAlgorithm::None,
false,
)),
];
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, data.len(), data.len()).await;
assert_eq!(written, 0);
let err = err.expect("inconsistent parity sources must fail the read");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert!(err.to_string().contains("inconsistent read source shards"));
assert!(output.is_empty());
}
async fn run_decode_midway_death_case(hash_algo: HashAlgorithm, context: &str) {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_size = hash_algo.size();
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
while offset < total_len {
let end = (offset + BLOCK_SIZE).min(total_len);
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
offset = end;
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let first_block_len = (hash_size + shard_size).min(shard_bufs[0].len());
let (readers, handles) =
readers_with_deferred_parity(&shard_bufs, DATA_SHARDS, shard_size, &hash_algo, &[(0, first_block_len)]);
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_stripe_handles(&mut output, readers, 0, total_len, total_len, None, handles)
.await;
assert!(
err.is_none(),
"{context}, algo={hash_algo:?}: mid-object data-shard failure must still reconstruct: {err:?}"
);
assert_eq!(
written, total_len,
"{context}, algo={hash_algo:?}: short write after mid-object shard failure"
);
assert_eq!(
output, total_data,
"{context}, algo={hash_algo:?}: reconstructed bytes mismatch (stripe desync?)"
);
}
#[tokio::test]
#[serial_test::serial]
async fn test_erasure_decode_recovers_when_data_shard_dies_midway() {
for enabled in [None, Some("true")] {
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, enabled)], async {
let context = if enabled.is_some() { "data-shards-only" } else { "default" };
for hash_algo in [HashAlgorithm::HighwayHash256, HashAlgorithm::None] {
run_decode_midway_death_case(hash_algo, context).await;
}
})
.await;
}
}
async fn encode_prefetch_object(erasure: &Erasure, total_data: &[u8], hash_algo: &HashAlgorithm) -> Vec<Vec<u8>> {
let total_shards = erasure.data_shards + erasure.parity_shards;
let shard_size = erasure.shard_size();
let block_size = erasure.block_size;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
while offset < total_data.len() {
let end = (offset + block_size).min(total_data.len());
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
offset = end;
}
shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect()
}
fn build_prefetch_readers(
shard_bufs: &[Vec<u8>],
shard_size: usize,
hash_algo: &HashAlgorithm,
block_size: usize,
offset: usize,
missing: &[usize],
corrupt: &[usize],
) -> Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> {
let hash_size = hash_algo.size();
let start_block = offset / block_size;
let cursor_pos = start_block * (shard_size + hash_size);
shard_bufs
.iter()
.enumerate()
.map(|(i, buf)| {
if missing.contains(&i) {
return None;
}
let mut bytes = buf.clone();
if corrupt.contains(&i) {
let flip = cursor_pos + hash_size;
if flip < bytes.len() {
bytes[flip] ^= 0xFF;
}
}
let mut cursor = Cursor::new(bytes);
cursor.set_position(cursor_pos as u64);
Some(BitrotReader::new(cursor, shard_size, hash_algo.clone(), false))
})
.collect()
}
type PrefetchEnvConfig = (&'static str, Vec<(&'static str, Option<&'static str>)>);
fn prefetch_env_configs() -> Vec<PrefetchEnvConfig> {
vec![
(
"serial-default",
vec![
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, None),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, None),
],
),
(
"prefetch-count-2",
vec![
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("2")),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, None),
],
),
(
"bitrot-overlap",
vec![
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, None),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true")),
],
),
(
"prefetch-count-8",
vec![
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("8")),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true")),
],
),
]
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_full_and_range_reads_match_serial() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
let cases: &[(usize, usize, &str)] = &[
(0, total_len, "full read"),
(0, 50, "head partial block"),
(10, 30, "within first block"),
(60, 80, "crossing two block boundaries"),
(128, 50, "starting at block boundary"),
(130, 10, "small range deep in middle"),
(192, 8, "tail partial block"),
(0, 128, "two full stripes ending on block boundary"),
(64, 64, "single full interior stripe on boundaries"),
];
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
for &(off, len, desc) in cases {
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, off, &[], &[]);
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, off, len, total_len).await;
assert!(err.is_none(), "{label}/{desc}: unexpected error {err:?}");
assert_eq!(written, len, "{label}/{desc}: written mismatch");
assert_eq!(output, total_data[off..off + len], "{label}/{desc}: bytes mismatch");
}
})
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_degraded_read_reconstructs() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[0, 2], &[]);
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
assert!(err.is_none(), "{label}: degraded read errored {err:?}");
assert_eq!(written, total_len, "{label}: degraded short write");
assert_eq!(output, total_data, "{label}: degraded bytes mismatch");
})
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_rejects_corrupt_shard_but_recovers() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[1]);
let mut output = Vec::new();
let outcome = erasure
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
&mut output,
readers,
0,
total_len,
total_len,
None,
Vec::new(),
Vec::new(),
)
.await;
assert!(outcome.error.is_none(), "{label}: corrupt-shard read errored {:?}", outcome.error);
assert_eq!(outcome.written, total_len, "{label}: corrupt-shard short write");
assert_eq!(output, total_data, "{label}: corrupt bytes leaked into output");
assert!(
outcome.repair.needed(false),
"{label}: minority corruption must survive successful reconstruction"
);
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[]);
let mut healthy_output = Vec::new();
let healthy = erasure
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
&mut healthy_output,
readers,
0,
total_len,
total_len,
None,
Vec::new(),
Vec::new(),
)
.await;
assert!(healthy.error.is_none());
assert_eq!(healthy_output, total_data);
assert!(
!healthy.repair.needed(true),
"{label}: healthy/deferred shards cannot create repair evidence"
);
})
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_unrecoverable_corruption_errors() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[0, 1], &[2]);
let mut output = Vec::new();
let (_written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
assert!(err.is_some(), "{label}: unrecoverable corruption must fail the read");
assert!(output.is_empty(), "{label}: emitted bytes despite an unrecoverable stripe 0");
})
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_attributes_late_stripe_failure() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let hash_size = hash_algo.size();
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
let first_block_len = hash_size + shard_size;
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = shard_bufs
.iter()
.enumerate()
.map(|(i, buf)| {
if i == 0 || i == 1 {
return None; }
let bytes = if i >= DATA_SHARDS {
buf[..first_block_len.min(buf.len())].to_vec() } else {
buf.clone()
};
Some(BitrotReader::new(Cursor::new(bytes), shard_size, hash_algo.clone(), false))
})
.collect();
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
assert!(err.is_some(), "{label}: truncated-parity object must fail on the later stripe");
assert_eq!(written, BLOCK_SIZE, "{label}: first stripe must emit before the later failure");
assert_eq!(output, total_data[..BLOCK_SIZE], "{label}: first stripe bytes wrong");
})
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_legacy_prefetch_no_hash_passthrough() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::None;
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
for (label, vars) in prefetch_env_configs() {
temp_env::async_with_vars(vars, async {
for &(off, len) in &[(0usize, total_len), (70usize, 90usize)] {
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, off, &[], &[]);
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, off, len, total_len).await;
assert!(err.is_none(), "{label}: no-hash read errored {err:?}");
assert_eq!(written, len, "{label}: no-hash written mismatch");
assert_eq!(output, total_data[off..off + len], "{label}: no-hash bytes mismatch");
}
})
.await;
}
}
#[test]
#[serial_test::serial]
fn test_legacy_stripe_prefetch_gate_defaults_off() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, None::<&str>),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, None::<&str>),
],
|| assert!(!legacy_stripe_prefetch_enabled(), "prefetch must default off"),
);
temp_env::with_vars([(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("2"))], || {
assert!(legacy_stripe_prefetch_enabled(), "count > 1 must enable prefetch");
});
temp_env::with_vars([(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true"))], || {
assert!(legacy_stripe_prefetch_enabled(), "overlap switch must enable prefetch");
});
temp_env::with_vars([(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("1"))], || {
assert!(!legacy_stripe_prefetch_enabled(), "count == 1 must stay serial");
});
}
#[tokio::test]
#[serial_test::serial]
async fn demand_bound_policy_disables_stripe_read_ahead_and_uses_data_only_lockstep() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("8")),
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("false")),
],
async {
assert!(legacy_stripe_prefetch_enabled());
assert!(!get_lockstep_data_shards_only_enabled());
with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
assert!(!legacy_stripe_prefetch_enabled());
assert!(get_lockstep_data_shards_only_enabled());
})
.await;
assert!(legacy_stripe_prefetch_enabled());
assert!(!get_lockstep_data_shards_only_enabled());
},
)
.await;
}
#[tokio::test(start_paused = true)]
#[serial_test::serial]
async fn test_legacy_prefetch_cancels_next_read_on_emit_failure() {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..(2 * BLOCK_SIZE) as u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let hash_size = hash_algo.size();
let shard_bufs = encode_prefetch_object(&erasure, &total_data, &hash_algo).await;
let first_block_len = hash_size + shard_size;
const READ_TIMEOUT_SECS: &str = "600";
const STALL: Duration = Duration::from_secs(3600);
const ASSERT_WINDOW: Duration = Duration::from_secs(5);
let vars = [
(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true")),
(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some(READ_TIMEOUT_SECS)),
];
temp_env::async_with_vars(vars, async {
let stall_polls = Arc::new(AtomicUsize::new(0));
let readers: Vec<Option<BitrotReader<TestShardReader>>> = shard_bufs
.iter()
.map(|buf| {
let prefix = buf[..first_block_len.min(buf.len())].to_vec();
let reader = TestShardReader::PrefixThenSlow {
cursor: Cursor::new(prefix),
stall: STALL,
sleep: None,
stall_polls: Arc::clone(&stall_polls),
};
Some(BitrotReader::new(reader, shard_size, hash_algo.clone(), false))
})
.collect();
let mut writer = YieldOnceThenFailWriter { yielded: false };
let start = TokioInstant::now();
let (written, err) = erasure.decode(&mut writer, readers, 0, total_len, total_len).await;
let elapsed = start.elapsed();
assert!(err.is_some(), "emit failure must surface as an error");
assert_eq!(written, 0, "the failing writer accepts no bytes");
assert!(
stall_polls.load(Ordering::SeqCst) > 0,
"the speculative next-stripe read must be in flight before emit fails"
);
assert!(
elapsed < ASSERT_WINDOW,
"prefetch next-stripe read was not cancelled on emit failure: took {elapsed:?} \
(>= {ASSERT_WINDOW:?}); a tokio::join! regression waits out the shard read timeout"
);
})
.await;
}
async fn healthy_lockstep_shard_bytes() -> (usize, Vec<usize>) {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
while offset < total_len {
let end = (offset + BLOCK_SIZE).min(total_len);
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
offset = end;
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let counters: Vec<Arc<AtomicUsize>> = (0..total_shards).map(|_| Arc::new(AtomicUsize::new(0))).collect();
let readers: Vec<Option<BitrotReader<CountingShardReader>>> = shard_bufs
.iter()
.enumerate()
.map(|(i, buf)| {
Some(BitrotReader::new(
CountingShardReader {
inner: Cursor::new(buf.clone()),
bytes_read: Arc::clone(&counters[i]),
},
shard_size,
hash_algo.clone(),
false,
))
})
.collect();
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
assert!(err.is_none(), "healthy decode must succeed: {err:?}");
assert_eq!(written, total_len);
assert_eq!(output, total_data);
(DATA_SHARDS, counters.iter().map(|counter| counter.load(Ordering::SeqCst)).collect())
}
#[tokio::test]
#[serial_test::serial]
async fn test_lockstep_healthy_get_reads_only_data_shards() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async {
let (data_shards, shard_bytes) = healthy_lockstep_shard_bytes().await;
for (i, bytes) in shard_bytes.iter().enumerate() {
if i < data_shards {
assert!(*bytes > 0, "data shard {i} must be read on the healthy path");
} else {
assert_eq!(
*bytes, 0,
"healthy GET must not read parity shard {i} (2x read amplification, backlog#923)"
);
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_lockstep_default_reads_all_shards_per_stripe() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, None::<&str>)], async {
let (_data_shards, shard_bytes) = healthy_lockstep_shard_bytes().await;
for (i, bytes) in shard_bytes.iter().enumerate() {
assert!(*bytes > 0, "default lockstep behavior must read shard {i} on every stripe");
}
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_erasure_decode_rejects_inconsistent_parity_engaged_midstream() {
for enabled in [None, Some("true")] {
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, enabled)], async {
run_inconsistent_parity_midstream_case(if enabled.is_some() { "data-shards-only" } else { "default" }).await;
})
.await;
}
}
async fn run_inconsistent_parity_midstream_case(context: &str) {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let total_len = total_data.len();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let hash_size = hash_algo.size();
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let mut offset = 0;
let mut stripe = 0usize;
while offset < total_len {
let end = (offset + BLOCK_SIZE).min(total_len);
let shards = erasure.encode_data(&total_data[offset..end]).unwrap();
for (i, shard) in shards.iter().enumerate() {
if i == DATA_SHARDS && stripe >= 1 {
let mut corrupted = shard.to_vec();
corrupted[0] ^= 0x80;
shard_writers[i].write(&corrupted).await.unwrap();
} else {
shard_writers[i].write(shard).await.unwrap();
}
}
offset = end;
stripe += 1;
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let first_block_len = (hash_size + shard_size).min(shard_bufs[0].len());
let (readers, handles) =
readers_with_deferred_parity(&shard_bufs, DATA_SHARDS, shard_size, &hash_algo, &[(0, first_block_len)]);
let mut output = Vec::new();
let (_written, err) = erasure
.decode_with_stripe_handles(&mut output, readers, 0, total_len, total_len, None, handles)
.await;
let err = err.unwrap_or_else(|| panic!("{context}: inconsistent parity engaged mid-stream must fail the read"));
assert_eq!(err.kind(), ErrorKind::InvalidData, "{context}");
assert!(err.to_string().contains("inconsistent read source shards"), "{context}: {err}");
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_erasure_decode_preserves_compressed_stream_near_block_boundary() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 1024 * 1024;
use crate::io_support::rio::CompressReader;
use rustfs_utils::CompressionAlgorithm;
use tokio::io::AsyncReadExt;
let plaintext_size = 8 * BLOCK_SIZE + 123;
let plaintext = (0..plaintext_size)
.scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
*state ^= *state << 7;
*state ^= *state >> 9;
*state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
Some((*state >> 32) as u8)
})
.collect::<Vec<_>>();
let mut compressor = CompressReader::new(Cursor::new(plaintext), CompressionAlgorithm::default());
let mut compressed = Vec::new();
compressor.read_to_end(&mut compressed).await.unwrap();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
for block in compressed.chunks(BLOCK_SIZE) {
let shards = erasure.encode_data(block).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let mut decoded = Vec::new();
let (written, err) = erasure
.decode(&mut decoded, readers, 0, compressed.len(), compressed.len())
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, compressed.len());
assert_eq!(decoded, compressed);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_preference_gate_defaults_disabled() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_mode_parses_supported_values() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Observe);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
assert!(should_collect_shard_read_costs());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("unexpected")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_off_does_not_collect_without_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("off")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_observe_collects_only_with_stage_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
assert!(should_collect_shard_read_costs());
assert!(!get_shard_locality_preference_enabled());
let read_costs = [ShardReadCost::Remote, ShardReadCost::Local, ShardReadCost::SameNode];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2]);
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_on_enables_reordering() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_legacy_preference_gate_still_enables_on() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
fn test_shard_locality_read_launch_order_is_gated() {
let read_costs = [
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Unknown,
ShardReadCost::SameNode,
ShardReadCost::Remote,
];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2, 3, 4]);
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 0, 4, 2]);
}
#[test]
fn test_shard_locality_remote_avoid_potential_is_observe_only() {
assert_eq!(shard_locality_remote_avoid_potential(2, 4, 4), 2);
assert_eq!(shard_locality_remote_avoid_potential(2, 2, 4), 0);
assert_eq!(shard_locality_remote_avoid_potential(3, 3, 4), 2);
}
#[test]
#[serial_test::serial]
fn parallel_reader_drops_metrics_path_when_stage_metrics_disabled() {
let erasure = Erasure::new(2, 1, 32);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
let reader = ParallelReader::new_with_metrics_path(readers, erasure.clone(), 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
assert_eq!(reader.metrics_path, None);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let reader = ParallelReader::new_with_metrics_path(readers, erasure, 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
assert_eq!(reader.metrics_path, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
}
#[tokio::test]
#[serial_test::serial]
async fn parallel_reader_records_metrics_for_observe_and_locality_policy_modes() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
for (mode, expected_data_slots) in [("off", vec![0, 1, 2, 3]), ("on", vec![2, 3, 4, 5])] {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some(mode)),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let hash_algo = HashAlgorithm::HighwayHash256;
let readers =
make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
Some(GET_OBJECT_PATH_LEGACY_DUPLEX),
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(parallel_reader.metrics_path, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
assert!(errs.iter().all(Option::is_none));
for index in expected_data_slots {
assert_eq!(bufs[index].as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
},
)
.await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[0].is_none());
assert!(bufs[1].is_none());
for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
assert!(errs.iter().all(Option::is_none));
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_missing_falls_back_to_remote() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(matches!(errs[1], Some(Error::FileNotFound)));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_corrupt_falls_back_to_remote() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert_eq!(errs[1], Some(DiskError::FileCorrupt));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_erasure_decode_local_first_preserves_output_order() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::SameNode,
];
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, total_data.len());
assert_eq!(output, total_data);
},
)
.await;
}
#[tokio::test]
async fn test_parallel_reader_normal() {
const BLOCK_SIZE: usize = 64;
const NUM_SHARDS: usize = 2;
const DATA_SHARDS: usize = 8;
const PARITY_SHARDS: usize = 4;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let reader_offset = 0;
let mut readers = vec![];
for i in 0..(DATA_SHARDS + PARITY_SHARDS) {
readers.push(Some(
create_reader(SHARD_SIZE, NUM_SHARDS, (i % 256) as u8, &HashAlgorithm::HighwayHash256, false).await,
));
}
let erausre = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new(readers, erausre, reader_offset, NUM_SHARDS * BLOCK_SIZE);
for _ in 0..NUM_SHARDS {
let (bufs, errs) = parallel_reader.read().await;
bufs.into_iter().enumerate().for_each(|(index, buf)| {
if index < DATA_SHARDS {
assert!(buf.is_some());
let buf = buf.unwrap();
assert_eq!(SHARD_SIZE, buf.len());
assert_eq!(index as u8, buf[0]);
} else {
assert!(buf.is_none());
}
});
assert!(errs.iter().filter(|err| err.is_some()).count() == 0);
}
}
#[tokio::test]
async fn test_parallel_reader_with_offline_disks() {
const OFFLINE_DISKS: usize = 2;
const NUM_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 8;
const PARITY_SHARDS: usize = 4;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let reader_offset = 0;
let mut readers = vec![];
for i in 0..(DATA_SHARDS + PARITY_SHARDS) {
if i < OFFLINE_DISKS {
readers.push(None);
} else {
readers.push(Some(
create_reader(SHARD_SIZE, NUM_SHARDS, (i % 256) as u8, &HashAlgorithm::HighwayHash256, false).await,
));
}
}
let erausre = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new(readers, erausre, reader_offset, NUM_SHARDS * BLOCK_SIZE);
for _ in 0..NUM_SHARDS {
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert_eq!(OFFLINE_DISKS, errs.iter().filter(|err| err.is_some()).count());
}
}
#[tokio::test]
async fn test_parallel_reader_with_bitrots() {
const BITROT_DISKS: usize = 2;
const NUM_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 8;
const PARITY_SHARDS: usize = 4;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let reader_offset = 0;
let mut readers = vec![];
for i in 0..(DATA_SHARDS + PARITY_SHARDS) {
readers.push(Some(
create_reader(SHARD_SIZE, NUM_SHARDS, (i % 256) as u8, &HashAlgorithm::HighwayHash256, i < BITROT_DISKS).await,
));
}
let erausre = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new(readers, erausre, reader_offset, NUM_SHARDS * BLOCK_SIZE);
for _ in 0..NUM_SHARDS {
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert_eq!(
BITROT_DISKS,
errs.iter().filter(|err| matches!(err, Some(DiskError::FileCorrupt))).count()
);
}
}
#[tokio::test]
async fn test_parallel_reader_replaces_timed_out_shard_with_parity() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 1;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::TimedOut, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new(readers, erasure, 0, NUM_SHARDS * BLOCK_SIZE);
let (bufs, errs) = parallel_reader.read().await;
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test]
async fn test_parallel_reader_schedules_extra_parity_for_reconstruction_verification() {
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
None,
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader =
ParallelReader::new_with_metrics_path_and_reconstruction_verification(readers, erasure, 0, BLOCK_SIZE, None);
let (bufs, errs) = parallel_reader.read().await;
assert!(errs[0].is_none() || matches!(&errs[0], Some(DiskError::FileNotFound)));
assert!(bufs[0].is_none());
assert_eq!(3, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert!(bufs[3].is_some());
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_records_metrics_for_success_missing_error_and_timeout() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(TestShardReader::TimedOut, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
None,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
Some(GET_OBJECT_PATH_LEGACY_DUPLEX),
vec![
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Remote,
ShardReadCost::Unknown,
],
Duration::from_millis(20),
false,
);
let started = std::time::Instant::now();
let (bufs, errs) = parallel_reader.read().await;
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(started.elapsed() < Duration::from_secs(1));
assert_eq!(parallel_reader.metrics_path, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(matches!(&errs[1], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(matches!(&errs[3], Some(Error::FileNotFound)));
assert_eq!(bufs[2].as_deref(), Some(&[2_u8; SHARD_SIZE][..]));
}
#[tokio::test]
async fn test_parallel_reader_uses_parity_without_waiting_for_pending_shard() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 1;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader =
ParallelReader::new_with_read_timeout(readers, erasure, 0, NUM_SHARDS * BLOCK_SIZE, Duration::from_secs(60));
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("parallel reader should use ready parity without waiting for pending data shard");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test]
async fn test_lockstep_hedges_slow_shard_instead_of_waiting_read_timeout() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
vec![ShardReadCost::Unknown; 4],
Duration::from_secs(60),
true,
);
assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("lockstep read must hedge the slow shard instead of waiting out the 60s read timeout");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert!(bufs[3].is_some());
assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test]
async fn test_lockstep_hedges_shard_that_stops_mid_read() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(
TestShardReader::PartialThenPending {
data: vec![0xab],
emitted: false,
},
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
vec![ShardReadCost::Unknown; 4],
Duration::from_secs(60),
true,
);
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("a shard that stops mid-read must be hedged instead of waiting out the 60s read timeout");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test(start_paused = true)]
async fn lockstep_reopens_hedged_shard_after_later_peer_loss() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let data: Vec<u8> = (0..192).collect();
let mut shard_bytes = vec![Vec::new(); DATA_SHARDS + PARITY_SHARDS];
for block in data.chunks(BLOCK_SIZE) {
for (bytes, encoded) in shard_bytes
.iter_mut()
.zip(erasure.encode_data(block).expect("encode test stripe"))
{
bytes.extend_from_slice(&encoded);
}
}
let mut readers: Vec<_> = shard_bytes
.iter()
.map(|bytes| {
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(bytes.clone())),
SHARD_SIZE,
HashAlgorithm::None,
false,
))
})
.collect();
readers[0] = Some(BitrotReader::new(TestShardReader::Parked, SHARD_SIZE, HashAlgorithm::None, false));
let reopen_calls = Arc::new(AtomicUsize::new(0));
let mut reopeners: Vec<Option<DeferredReaderReopener<TestShardReader>>> = vec![None; readers.len()];
let reopened_bytes = shard_bytes[0].clone();
let calls = Arc::clone(&reopen_calls);
reopeners[0] = Some(Arc::new(move |stripe| {
calls.fetch_add(1, Ordering::SeqCst);
let remaining = reopened_bytes.get(stripe * SHARD_SIZE..)?.to_vec();
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(remaining)),
SHARD_SIZE,
HashAlgorithm::None,
false,
))
}));
for slot in &mut reopeners[1..3] {
*slot = Some(Arc::new(|_| panic!("a real peer read error must not be retried")));
}
let mut reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure.clone(),
0,
data.len(),
None,
vec![ShardReadCost::Unknown; 4],
Duration::from_secs(60),
true,
)
.with_deferred_parity_reopeners(reopeners);
let (first, errors) = reader.read().await;
assert_eq!(first.iter().flatten().count(), DATA_SHARDS + 1);
assert!(matches!(&errors[0], Some(DiskError::Io(error)) if error.kind() == ErrorKind::TimedOut));
assert_eq!(reopen_calls.load(Ordering::SeqCst), 0, "a healthy stripe must keep its latency hedge");
for slot in &mut reader.readers[1..3] {
*slot = Some(BitrotReader::new(
TestShardReader::TerminalFileNotFound,
SHARD_SIZE,
HashAlgorithm::None,
false,
));
}
for stripe in 1..3 {
let (mut shards, _) = reader.read().await;
assert_eq!(
shards.iter().flatten().count(),
DATA_SHARDS,
"surviving physical shards must remain usable"
);
assert_eq!(
shards[0].as_deref(),
Some(&shard_bytes[0][stripe * SHARD_SIZE..(stripe + 1) * SHARD_SIZE])
);
erasure
.decode_data_with_reconstruction_verification(&mut shards)
.expect("recover the stripe after peer loss");
let actual: Vec<u8> = shards[..DATA_SHARDS].iter().flatten().flatten().copied().collect();
assert_eq!(actual, data[stripe * BLOCK_SIZE..(stripe + 1) * BLOCK_SIZE]);
}
assert_eq!(
reopen_calls.load(Ordering::SeqCst),
1,
"a reopened healthy reader must stay aligned across stripes"
);
}
#[tokio::test]
async fn test_demand_bound_lockstep_hedges_to_deferred_parity_quorum() {
with_decode_read_policy(DecodeReadPolicy::DemandBound, assert_deferred_parity_hedges_slow_data()).await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_data_shards_only_gate_hedges_to_deferred_parity_quorum() {
temp_env::async_with_vars(
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
assert_deferred_parity_hedges_slow_data(),
)
.await;
}
async fn assert_deferred_parity_hedges_slow_data() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let slow_until = TokioInstant::now() + Duration::from_secs(60);
let readers = vec![
Some(BitrotReader::new(
TestShardReader::ReadyAt {
cursor: Cursor::new(vec![0_u8; SHARD_SIZE * NUM_SHARDS]),
ready_at: slow_until,
sleep: None,
},
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
Duration::from_secs(60),
true,
);
let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
.await
.expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1);
assert_eq!(parallel_reader.engaged.as_slice(), &[true, true, true, true]);
assert_eq!(
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
vec![false, true, true, true]
);
}
#[tokio::test]
async fn test_demand_bound_admits_parity_after_fast_data_failure_without_timer() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::TimedOut, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let (bufs, errs, engaged, readers_remaining) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
Duration::ZERO,
true,
);
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("fast data failure must immediately race healthy parity without a hedge timer");
(
bufs,
errs,
parallel_reader.engaged.clone(),
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
)
})
.await;
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1);
assert_eq!(engaged.as_slice(), &[true, true, true, true]);
assert_eq!(readers_remaining, vec![false, true, true, true]);
}
#[tokio::test]
async fn test_demand_bound_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
with_decode_read_policy(
DecodeReadPolicy::DemandBound,
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_data_shards_only_gate_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
temp_env::async_with_vars(
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
)
.await;
}
async fn assert_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let parity_calls = Arc::new(AtomicUsize::new(0));
let mut reopeners: Vec<Option<DeferredReaderReopener<TestShardReader>>> = vec![None; DATA_SHARDS + PARITY_SHARDS];
for (idx, slot) in reopeners.iter_mut().enumerate().skip(DATA_SHARDS).take(PARITY_SHARDS) {
let parity_calls = Arc::clone(&parity_calls);
*slot = Some(Arc::new(move |_stripe_index| {
let call = parity_calls.fetch_add(1, Ordering::SeqCst);
let reader = if call == 0 {
TestShardReader::Ready(Cursor::new(vec![idx as u8; SHARD_SIZE]))
} else if call < PARITY_SHARDS {
TestShardReader::Parked
} else {
TestShardReader::Ready(Cursor::new(vec![idx as u8; SHARD_SIZE * 2]))
};
Some(BitrotReader::new(reader, SHARD_SIZE, HashAlgorithm::None, false))
}));
}
let slow_until = TokioInstant::now() + Duration::from_millis(150);
let readers = vec![
Some(BitrotReader::new(
TestShardReader::ReadyAt {
cursor: Cursor::new(vec![0_u8; SHARD_SIZE * 4]),
ready_at: slow_until,
sleep: None,
},
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * 4])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo, false)),
];
let (first_parity_reserved, second_result) = {
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification(
readers,
erasure,
0,
BLOCK_SIZE * 4,
None,
Duration::from_secs(1),
true,
)
.with_deferred_parity_reopeners(reopeners);
let (first_buffers, first_errors) = tokio::time::timeout(Duration::from_secs(1), parallel_reader.read())
.await
.expect("healthy data recovery must not wait for canceled parity hedges");
assert_eq!(first_buffers.iter().filter(|buffer| buffer.is_some()).count(), DATA_SHARDS + 1);
assert!(first_errors.iter().take(DATA_SHARDS).all(Option::is_none));
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS);
assert!(parallel_reader.readers[2].is_some());
assert!(parallel_reader.readers[3].is_some());
assert!(!parallel_reader.engaged[2]);
assert!(!parallel_reader.engaged[3]);
let (healthy_buffers, healthy_errors) = tokio::time::timeout(Duration::from_secs(1), parallel_reader.read())
.await
.expect("a healthy following stripe must complete without parity fan-out");
assert!(healthy_errors.iter().take(DATA_SHARDS).all(Option::is_none));
assert_eq!(healthy_buffers.iter().filter(|buffer| buffer.is_some()).count(), DATA_SHARDS);
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS);
assert!(!parallel_reader.engaged[2]);
assert!(!parallel_reader.engaged[3]);
parallel_reader.readers[0] =
Some(BitrotReader::new(TestShardReader::TimedOut, SHARD_SIZE, HashAlgorithm::None, false));
let (second_buffers, second_errors) = tokio::time::timeout(Duration::from_secs(1), parallel_reader.read())
.await
.expect("the next degraded stripe must reuse the preserved parity reserve");
assert!(matches!(&second_errors[0], Some(DiskError::Io(error)) if error.kind() == ErrorKind::TimedOut));
assert_eq!(second_buffers.iter().filter(|buffer| buffer.is_some()).count(), DATA_SHARDS + 1);
let (third_buffers, third_errors) = tokio::time::timeout(Duration::from_secs(1), parallel_reader.read())
.await
.expect("persistent parity readers must cover a subsequent degraded stripe");
assert!(third_errors[0].is_none(), "an already-retired data slot has no new read error");
assert_eq!(third_buffers.iter().filter(|buffer| buffer.is_some()).count(), DATA_SHARDS + 1);
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2);
(
parallel_reader.readers[2].is_some() && parallel_reader.readers[3].is_some(),
(third_buffers, third_errors),
)
};
assert!(first_parity_reserved);
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2);
let (buffers, errors) = second_result;
assert!(errors[0].is_none());
assert_eq!(buffers.iter().filter(|buffer| buffer.is_some()).count(), DATA_SHARDS + 1);
}
#[tokio::test]
async fn test_demand_bound_disposable_parity_error_does_not_poison_recovered_stripe() {
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let parity_calls = Arc::new(AtomicUsize::new(0));
let mut reopeners: Vec<Option<DeferredReaderReopener<TestShardReader>>> = vec![None; DATA_SHARDS + PARITY_SHARDS];
for (idx, slot) in reopeners.iter_mut().enumerate().skip(DATA_SHARDS).take(PARITY_SHARDS) {
let parity_calls = Arc::clone(&parity_calls);
*slot = Some(Arc::new(move |_stripe_index| {
let call = parity_calls.fetch_add(1, Ordering::SeqCst);
let reader = if call == 0 {
TestShardReader::TerminalFileNotFound
} else {
TestShardReader::Ready(Cursor::new(vec![idx as u8; SHARD_SIZE]))
};
Some(BitrotReader::new(reader, SHARD_SIZE, HashAlgorithm::None, false))
}));
}
let slow_until = TokioInstant::now() + Duration::from_millis(150);
let readers = vec![
Some(BitrotReader::new(
TestShardReader::ReadyAt {
cursor: Cursor::new(vec![0_u8; SHARD_SIZE]),
ready_at: slow_until,
sleep: None,
},
SHARD_SIZE,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE])),
SHARD_SIZE,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, HashAlgorithm::None, false)),
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, HashAlgorithm::None, false)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut output = Vec::new();
let (written, error) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
erasure
.decode_with_stripe_handles_and_reopeners(
&mut output,
readers,
0,
BLOCK_SIZE,
BLOCK_SIZE,
None,
Vec::new(),
reopeners,
)
.await
})
.await;
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS);
assert_eq!(written, BLOCK_SIZE);
assert_eq!(output.len(), BLOCK_SIZE);
assert!(error.is_none(), "a failed disposable hedge must not fail a recovered stripe: {error:?}");
}
#[tokio::test]
#[serial_test::serial]
async fn test_data_shards_only_gate_data_and_parity_failure_fails_before_output() {
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async {
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let payload = (0..BLOCK_SIZE).map(|value| value as u8).collect::<Vec<_>>();
let shards = erasure.encode_data(&payload).expect("test payload should encode");
let shard_size = erasure.shard_size();
let readers = vec![
Some(BitrotReader::new(TestShardReader::TimedOut, shard_size, HashAlgorithm::None, false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(shards[1].to_vec())),
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
TestShardReader::TerminalFileNotFound,
shard_size,
HashAlgorithm::None,
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(shards[3].to_vec())),
shard_size,
HashAlgorithm::None,
false,
)),
];
let mut output = Vec::new();
let (written, error) = erasure.decode(&mut output, readers, 0, payload.len(), payload.len()).await;
assert_eq!(written, 0, "an unverified stripe must not report body bytes");
assert!(output.is_empty(), "an unverified stripe must not expose a clean short body");
let error = error.expect("data plus parity loss must fail closed");
assert_eq!(error.kind(), ErrorKind::InvalidData);
assert!(error.to_string().contains("insufficient source shards"));
})
.await;
}
#[tokio::test]
async fn test_lockstep_hedge_waits_for_verification_quorum() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let parity3_ready_at = TokioInstant::now() + Duration::from_millis(200);
let readers = vec![
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::ReadyAt {
cursor: Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS]),
ready_at: parity3_ready_at,
sleep: None,
},
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
vec![ShardReadCost::Unknown; 4],
Duration::from_secs(60),
true,
);
assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("hedge must reach data_shards + 1, not stall to the 60s read timeout");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert!(bufs[3].is_some(), "verification-source parity must be used, not settled away");
assert!(parallel_reader.readers[3].is_some(), "verification-source parity must not be retired");
assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test]
async fn test_parallel_reader_drains_completed_shards_after_quorum() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 3;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let ready_at = TokioInstant::now() + Duration::from_millis(250);
let readers = (0..DATA_SHARDS + PARITY_SHARDS)
.map(|index| {
Some(BitrotReader::new(
TestShardReader::ReadyAt {
cursor: Cursor::new(vec![index as u8; SHARD_SIZE * NUM_SHARDS]),
ready_at,
sleep: None,
},
SHARD_SIZE,
HashAlgorithm::None,
false,
))
})
.collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader =
ParallelReader::new_with_read_timeout(readers, erasure, 0, NUM_SHARDS * BLOCK_SIZE, Duration::from_secs(2));
let (bufs, errs) = parallel_reader.read().await;
assert!(errs.iter().all(Option::is_none));
assert!(bufs.iter().filter(|buf| buf.is_some()).count() >= DATA_SHARDS);
}
#[tokio::test]
async fn test_parallel_reader_retires_partially_read_shard_after_quorum() {
const NUM_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 1;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(
TestShardReader::PartialThenPending {
data: vec![9_u8; SHARD_SIZE / 2],
emitted: false,
},
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo.clone(),
false,
)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader =
ParallelReader::new_with_read_timeout(readers, erasure, 0, NUM_SHARDS * BLOCK_SIZE, Duration::from_secs(60));
let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("parallel reader should use parity without waiting for a partially read shard");
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_some());
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
let (next_bufs, next_errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
.await
.expect("retired partially read shard should not block the next stripe");
assert!(matches!(&next_errs[0], Some(DiskError::FileNotFound)));
assert!(next_bufs[0].is_none());
assert!(next_bufs[1].is_some());
assert!(next_bufs[2].is_some());
assert_eq!(DATA_SHARDS, next_bufs.iter().filter(|buf| buf.is_some()).count());
}
#[tokio::test]
async fn test_parallel_reader_retires_pending_shard_when_quorum_needs_it() {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 1;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::None;
let readers = vec![
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
Some(BitrotReader::new(
TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
SHARD_SIZE,
hash_algo,
false,
)),
None,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader =
ParallelReader::new_with_read_timeout(readers, erasure, 0, NUM_SHARDS * BLOCK_SIZE, Duration::from_millis(20));
let started = std::time::Instant::now();
let (bufs, errs) = parallel_reader.read().await;
assert!(started.elapsed() < Duration::from_secs(1));
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
assert!(matches!(&errs[2], Some(DiskError::FileNotFound)));
assert!(parallel_reader.readers[0].is_none());
assert!(bufs[0].is_none());
assert!(bufs[1].is_some());
assert!(bufs[2].is_none());
assert!(bufs.iter().filter(|buf| buf.is_some()).count() < DATA_SHARDS);
}
#[test]
fn test_zero_read_timeout_disables_scheduled_hedging() {
assert_eq!(shard_read_hedge_delay(Duration::ZERO), None);
assert_eq!(shard_read_hedge_delay(Duration::from_millis(50)), Some(Duration::from_millis(50)));
assert_eq!(shard_read_hedge_delay(Duration::from_secs(60)), Some(Duration::from_millis(100)));
temp_env::with_var(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, None::<&str>, || {
assert_eq!(get_decode_stripe_prefetch_count(), DEFAULT_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT);
});
temp_env::with_var(ENV_RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT, Some("3"), || {
assert_eq!(get_decode_stripe_prefetch_count(), 3);
});
temp_env::with_var(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, None::<&str>, || {
assert_eq!(is_bitrot_decode_overlap_enabled(), DEFAULT_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE);
});
temp_env::with_var(ENV_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE, Some("true"), || {
assert!(is_bitrot_decode_overlap_enabled());
});
}
async fn create_reader(
shard_size: usize,
num_shards: usize,
value: u8,
hash_algo: &HashAlgorithm,
bitrot: bool,
) -> BitrotReader<Cursor<Vec<u8>>> {
let len = (hash_algo.size() + shard_size) * num_shards;
let buf = Cursor::new(vec![0u8; len]);
let mut writer = BitrotWriter::new(buf, shard_size, hash_algo.clone());
for _ in 0..num_shards {
writer.write(vec![value; shard_size].as_slice()).await.unwrap();
}
let mut buf = writer.into_inner().into_inner();
if bitrot {
for i in 0..num_shards {
buf[i * (hash_algo.size() + shard_size)] ^= 1;
}
}
let reader_cursor = Cursor::new(buf);
BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
}
async fn make_test_readers(
total_shards: usize,
shard_size: usize,
num_shards: usize,
hash_algo: &HashAlgorithm,
missing: &[usize],
corrupt: &[usize],
) -> Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> {
let mut readers = Vec::with_capacity(total_shards);
for index in 0..total_shards {
if missing.contains(&index) {
readers.push(None);
} else {
readers.push(Some(
create_reader(shard_size, num_shards, (index % 256) as u8, hash_algo, corrupt.contains(&index)).await,
));
}
}
readers
}
async fn encode_test_object(erasure: &Erasure, data: &[u8], shard_size: usize, hash_algo: &HashAlgorithm) -> Vec<Vec<u8>> {
let total_shards = erasure.data_shards + erasure.parity_shards;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
let shards = erasure.encode_data(data).unwrap();
for (index, shard) in shards.iter().enumerate() {
shard_writers[index].write(shard).await.unwrap();
}
shard_writers
.into_iter()
.map(|writer| writer.into_inner().into_inner())
.collect()
}
}