use crate::disk::disk_store::get_object_disk_read_timeout;
use crate::disk::error::{Error, Result};
use crate::erasure::coding::BitrotReader;
use crate::erasure::coding::BitrotWriterWrapper;
use crate::erasure::coding::encode::MultiWriter;
use bytes::Bytes;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use std::io;
use std::io::ErrorKind;
use std::time::Duration;
use tokio::io::AsyncRead;
use tracing::{trace, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
const EVENT_ERASURE_HEAL_STARTED: &str = "erasure_heal_started";
async fn read_heal_shards<R>(
readers: &mut [Option<BitrotReader<R>>],
shard_size: usize,
read_timeout: Duration,
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
where
R: AsyncRead + Unpin + Send + Sync,
{
let num_readers = readers.len();
let mut shards = vec![None; num_readers];
let mut errs = vec![None; num_readers];
let mut retire_readers = Vec::new();
if shard_size == 0 {
return (shards, errs);
}
{
let mut futures = FuturesUnordered::new();
for (index, reader) in readers.iter_mut().enumerate() {
let Some(reader) = reader else {
errs[index] = Some(Error::FileNotFound);
continue;
};
futures.push(Box::pin(async move {
let mut buf = vec![0; shard_size];
let read_result = if read_timeout.is_zero() {
reader.read(&mut buf).await
} else {
match tokio::time::timeout(read_timeout, reader.read(&mut buf)).await {
Ok(result) => result,
Err(_) => {
return (
index,
Err(Error::from(io::Error::new(ErrorKind::TimedOut, "heal shard read timed out"))),
true,
);
}
}
};
match read_result {
Ok(n) => {
buf.truncate(n);
(index, Ok(buf), false)
}
Err(err) => {
let should_retire = err.kind() == ErrorKind::TimedOut;
(index, Err(Error::from(err)), should_retire)
}
}
}));
}
while let Some((index, result, should_retire)) = futures.next().await {
match result {
Ok(shard) => {
shards[index] = Some(shard);
}
Err(err) => {
errs[index] = Some(err);
if should_retire {
retire_readers.push(index);
}
}
}
}
}
for index in retire_readers {
readers[index] = None;
warn!(shard_index = index, "retiring timed-out heal shard reader");
}
(shards, errs)
}
fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
writers
.take_retryable_internode_write_failure()
.map(|error| Error::RemoteClientUnavailable(error.to_string()))
.unwrap_or_else(|| error.into())
}
impl super::Erasure {
pub async fn heal<R>(
&self,
writers: &mut [Option<BitrotWriterWrapper>],
readers: Vec<Option<BitrotReader<R>>>,
total_length: usize,
_prefer: &[bool],
) -> Result<()>
where
R: AsyncRead + Unpin + Send + Sync,
{
trace!(
event = EVENT_ERASURE_HEAL_STARTED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_ERASURE,
writer_count = writers.len(),
reader_count = readers.len(),
total_length,
"Erasure heal started"
);
if writers.len() != self.parity_shards + self.data_shards {
return Err(Error::other("invalid argument"));
}
let mut readers = readers;
let start_block = 0;
let mut end_block = total_length / self.block_size;
if !total_length.is_multiple_of(self.block_size) {
end_block += 1;
}
let write_quorum = 1;
let mut writers = MultiWriter::new(writers, write_quorum);
let read_timeout = get_object_disk_read_timeout();
let shard_file_size = self.shard_file_size(total_length as i64) as usize;
for block_index in start_block..end_block {
let shard_offset = block_index * self.shard_size();
let shard_size = self.shard_size().min(shard_file_size.saturating_sub(shard_offset));
let (mut shards, errs) = read_heal_shards(&mut readers, shard_size, read_timeout).await;
let available_shards = errs.iter().filter(|e| e.is_none()).count();
if available_shards < self.data_shards {
warn!(
required_data_shards = self.data_shards,
available_shards,
total_shards = errs.len(),
errors = ?errs,
"Erasure heal read quorum unavailable"
);
return Err(Error::ErasureReadQuorum);
}
let source_parity = shards
.iter()
.enumerate()
.skip(self.data_shards)
.filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.clone())))
.collect::<Vec<_>>();
if self.parity_shards > 0 {
self.decode_data_and_parity(&mut shards)?;
if !source_parity.is_empty() && !self.verify_data_and_parity(&shards)? {
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
}
for (index, source) in source_parity {
let Some(rebuilt) = shards[index].as_ref() else {
return Err(Error::other("can not reconstruct data: missing rebuilt parity shard"));
};
if rebuilt != &source {
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
}
}
}
let shards = shards
.into_iter()
.map(|s| Bytes::from(s.unwrap_or_default()))
.collect::<Vec<_>>();
if let Err(error) = writers.write(shards).await {
return Err(heal_writer_failure(&mut writers, error));
}
}
if let Err(error) = writers.shutdown().await {
return Err(heal_writer_failure(&mut writers, error));
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::erasure::coding::{CustomWriter, Erasure};
use rustfs_utils::HashAlgorithm;
use std::io::{self, Cursor};
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncWrite, ReadBuf};
struct FailAfterBytes {
written: usize,
fail_at: usize,
}
impl AsyncWrite for FailAfterBytes {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
if self.written >= self.fail_at {
return Poll::Ready(Err(io::Error::other("injected heal write failure")));
}
self.written += buf.len();
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
struct InternodeFailureWriter {
fail_on_write: bool,
status: http::StatusCode,
}
impl InternodeFailureWriter {
fn error(&self) -> io::Error {
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
}
}
impl AsyncWrite for InternodeFailureWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
Poll::Ready(if self.fail_on_write {
Err(self.error())
} else {
Ok(buf.len())
})
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Err(self.error()))
}
}
struct PendingReader;
impl AsyncRead for PendingReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
}
struct FailingReader;
impl AsyncRead for FailingReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Err(io::Error::other("synthetic heal read failure")))
}
}
fn inline_writer(shard_size: usize) -> BitrotWriterWrapper {
BitrotWriterWrapper::new(CustomWriter::new_inline_buffer(), shard_size, HashAlgorithm::None)
}
#[tokio::test]
async fn read_heal_shards_returns_empty_slots_for_zero_sized_shards() {
let mut readers = vec![
Some(BitrotReader::new(Cursor::new(vec![1, 2, 3]), 0, HashAlgorithm::None, false)),
None,
];
let (shards, errs) = read_heal_shards(&mut readers, 0, Duration::ZERO).await;
assert_eq!(shards, vec![None, None]);
assert_eq!(errs, vec![None, None]);
}
#[tokio::test]
async fn read_heal_shards_records_reader_errors_and_retiring_timeouts() {
let mut readers = vec![Some(BitrotReader::new(FailingReader, 4, HashAlgorithm::None, false))];
let (shards, errs) = read_heal_shards(&mut readers, 4, Duration::ZERO).await;
assert!(shards[0].is_none());
assert!(
errs[0]
.as_ref()
.is_some_and(|err| err.to_string().contains("synthetic heal read failure"))
);
assert!(readers[0].is_some(), "ordinary read errors must not retire readers");
let mut timeout_readers = vec![Some(BitrotReader::new(PendingReader, 4, HashAlgorithm::None, false))];
let (shards, errs) = read_heal_shards(&mut timeout_readers, 4, Duration::from_millis(1)).await;
assert_eq!(shards, vec![None]);
assert!(errs[0].as_ref().is_some_and(|err| err.to_string().contains("timed out")));
assert!(timeout_readers[0].is_none(), "timed-out readers are retired");
}
#[tokio::test]
async fn heal_rejects_invalid_writer_shape_before_reading() {
let erasure = Erasure::new(2, 1, 64);
let mut writers = vec![Some(inline_writer(erasure.shard_size()))];
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = Vec::new();
let err = erasure
.heal(&mut writers, readers, 0, &[])
.await
.expect_err("invalid writer count must fail");
assert!(err.to_string().contains("invalid argument"));
}
#[tokio::test]
async fn heal_empty_object_only_shuts_down_available_writers() {
let erasure = Erasure::new(2, 1, 64);
let mut writers = (0..erasure.total_shard_count())
.map(|_| Some(inline_writer(erasure.shard_size())))
.collect::<Vec<_>>();
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = Vec::new();
erasure
.heal(&mut writers, readers, 0, &[])
.await
.expect("empty object heal should only close writers");
assert!(writers.iter().all(Option::is_some));
}
#[tokio::test]
async fn heal_maps_put_file_epoch_conflict_to_retryable_remote_unavailable() {
for status in [http::StatusCode::CONFLICT, http::StatusCode::BAD_REQUEST] {
for (fail_on_write, data) in [
(false, b"".as_slice()),
(false, b"payload".as_slice()),
(true, b"payload".as_slice()),
] {
let erasure = Erasure::new(2, 1, 64);
let encoded = erasure.encode_data(data).expect("source shards should encode");
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
(index < erasure.data_shards).then(|| {
BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false)
})
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
(index == erasure.data_shards).then(|| {
BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(InternodeFailureWriter { fail_on_write, status }),
erasure.shard_size(),
HashAlgorithm::None,
)
})
})
.collect::<Vec<_>>();
let error = erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect_err("failed sole target must not satisfy heal write quorum");
assert_eq!(
matches!(error, Error::RemoteClientUnavailable(_)),
status == http::StatusCode::CONFLICT,
"status={status}, fail_on_write={fail_on_write}, len={}, error={error:?}",
data.len()
);
assert!(writers.iter().all(Option::is_none), "failed target must not be committed");
}
}
}
#[tokio::test]
async fn heal_epoch_conflict_does_not_abort_healthy_target() {
for fail_on_write in [false, true] {
let erasure = Erasure::new(2, 2, 64);
let data = b"healthy target must retain exact reconstructed bytes";
let encoded = erasure.encode_data(data).expect("source shards should encode");
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
(index < erasure.data_shards)
.then(|| BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false))
})
.collect::<Vec<_>>();
let mut writers = vec![
None,
None,
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(InternodeFailureWriter {
fail_on_write,
status: http::StatusCode::CONFLICT,
}),
erasure.shard_size(),
HashAlgorithm::None,
)),
Some(inline_writer(erasure.shard_size())),
];
erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect("one healthy target must still satisfy the existing heal quorum");
assert!(writers[2].is_none(), "conflicting target must be dropped");
assert_eq!(
writers[3]
.take()
.expect("healthy target remains")
.into_inline_data()
.expect("inline target data"),
encoded[3].to_vec()
);
}
}
#[tokio::test]
async fn heal_reconstructs_missing_parity_shard() {
let erasure = Erasure::new(2, 2, 64);
let data = b"heal should write a rebuilt parity shard";
let encoded = erasure.encode_data(data).expect("encode should succeed");
let missing_parity = erasure.data_shards;
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index == missing_parity {
None
} else {
Some(BitrotReader::new(
Cursor::new(shard.to_vec()),
erasure.shard_size(),
HashAlgorithm::None,
false,
))
}
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
if index == missing_parity {
Some(inline_writer(erasure.shard_size()))
} else {
None
}
})
.collect::<Vec<_>>();
erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect("heal should rebuild parity");
let healed = writers[missing_parity]
.take()
.expect("parity writer should remain")
.into_inline_data()
.expect("inline writer should retain data");
assert_eq!(healed, encoded[missing_parity].to_vec());
}
#[tokio::test]
async fn heal_reconstructs_missing_data_shard_across_multiple_blocks() {
let erasure = Erasure::new(3, 2, 96);
let data = (0..erasure.block_size * 2 + 17)
.map(|index| (index % 251) as u8)
.collect::<Vec<_>>();
let encoded = erasure.encode_data(&data).expect("encode should succeed");
let missing_data = 1;
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index == missing_data {
None
} else {
Some(BitrotReader::new(
Cursor::new(shard.to_vec()),
erasure.shard_size(),
HashAlgorithm::None,
false,
))
}
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
if index == missing_data {
Some(inline_writer(erasure.shard_size()))
} else {
None
}
})
.collect::<Vec<_>>();
erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect("heal should rebuild data");
let healed = writers[missing_data]
.take()
.expect("data writer should remain")
.into_inline_data()
.expect("inline writer should retain data");
assert_eq!(healed, encoded[missing_data].to_vec());
}
#[tokio::test]
async fn heal_returns_read_quorum_when_available_shards_are_insufficient() {
let erasure = Erasure::new(3, 2, 64);
let data = b"heal should fail before decode when too few shards are readable";
let encoded = erasure.encode_data(data).expect("encode should succeed");
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index < 2 {
Some(BitrotReader::new(
Cursor::new(shard.to_vec()),
erasure.shard_size(),
HashAlgorithm::None,
false,
))
} else {
None
}
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count()).map(|_| None).collect::<Vec<_>>();
let err = erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect_err("heal should fail when available shards are below data shards");
assert!(matches!(err, Error::ErasureReadQuorum));
}
#[tokio::test]
async fn heal_rejects_inconsistent_sources_before_writing_data_shard() {
let erasure = Erasure::new(2, 2, 64);
let data = b"heal must not rebuild data from a stale parity shard";
let encoded = erasure.encode_data(data).expect("encode should succeed");
let missing_data = 1;
let corrupt_parity = erasure.data_shards;
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index == missing_data {
return None;
}
let mut shard = shard.to_vec();
if index == corrupt_parity {
shard[0] ^= 0x5a;
}
Some(BitrotReader::new(Cursor::new(shard), erasure.shard_size(), HashAlgorithm::None, false))
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
if index == missing_data {
Some(inline_writer(erasure.shard_size()))
} else {
None
}
})
.collect::<Vec<_>>();
let err = erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect_err("heal should reject inconsistent source shards");
assert!(err.to_string().contains("inconsistent heal source shards"));
let written = writers[missing_data]
.take()
.expect("data writer should remain")
.into_inline_data()
.expect("inline writer should retain data");
assert!(written.is_empty(), "heal must fail before writing rebuilt data");
}
#[tokio::test]
async fn heal_completes_healthy_targets_when_one_target_disk_fails_midway() {
let erasure = Erasure::new(2, 3, 64);
let data = (0..erasure.block_size * 3)
.map(|index| (index % 251) as u8)
.collect::<Vec<_>>();
let encoded = erasure.encode_data(&data).expect("encode should succeed");
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index < erasure.data_shards {
Some(BitrotReader::new(
Cursor::new(shard.to_vec()),
erasure.shard_size(),
HashAlgorithm::None,
false,
))
} else {
None
}
})
.collect::<Vec<_>>();
let failing_target = erasure.data_shards + 1;
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
if index < erasure.data_shards {
None
} else if index == failing_target {
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(FailAfterBytes {
written: 0,
fail_at: erasure.shard_size(),
}),
erasure.shard_size(),
HashAlgorithm::None,
))
} else {
Some(BitrotWriterWrapper::new(
CustomWriter::new_inline_buffer(),
erasure.shard_size(),
HashAlgorithm::None,
))
}
})
.collect::<Vec<_>>();
erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect("heal must succeed on the healthy targets despite one failing disk");
assert!(writers[failing_target].is_none(), "failing target writer must be dropped, not committed");
for index in erasure.data_shards..erasure.total_shard_count() {
if index == failing_target {
continue;
}
let healed = writers[index]
.take()
.expect("healthy target writer should remain")
.into_inline_data()
.expect("inline writer should retain data");
assert_eq!(healed, encoded[index].to_vec(), "healthy target {index} must be fully healed");
}
}
#[tokio::test]
async fn heal_fails_when_all_target_disks_fail_midway() {
let erasure = Erasure::new(2, 3, 64);
let data = (0..erasure.block_size * 3)
.map(|index| (index % 251) as u8)
.collect::<Vec<_>>();
let encoded = erasure.encode_data(&data).expect("encode should succeed");
let readers = encoded
.iter()
.enumerate()
.map(|(index, shard)| {
if index < erasure.data_shards {
Some(BitrotReader::new(
Cursor::new(shard.to_vec()),
erasure.shard_size(),
HashAlgorithm::None,
false,
))
} else {
None
}
})
.collect::<Vec<_>>();
let mut writers = (0..erasure.total_shard_count())
.map(|index| {
if index < erasure.data_shards {
None
} else {
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(FailAfterBytes {
written: 0,
fail_at: erasure.shard_size(),
}),
erasure.shard_size(),
HashAlgorithm::None,
))
}
})
.collect::<Vec<_>>();
erasure
.heal(&mut writers, readers, data.len(), &[])
.await
.expect_err("heal must fail when all replacement disks fail");
}
}