use super::super::{
Cursor, DiskStore, EVENT_SET_DISK_WRITE, Error, FileInfo, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK,
Result, join_all, warn,
};
use crate::disk::DiskAPI;
#[cfg(test)]
use crate::disk::RUSTFS_META_TMP_BUCKET;
use crate::set_disk::coding;
#[cfg(test)]
use bytes::Bytes;
pub(in crate::set_disk::ops) fn drop_failed_writer_disks<D, W>(disks: &mut [Option<D>], writers: &[Option<W>]) -> usize {
let mut committed = 0usize;
for (slot, writer) in disks.iter_mut().zip(writers.iter()) {
if writer.is_none() {
*slot = None;
} else if slot.is_some() {
committed += 1;
}
}
committed
}
#[derive(Clone, Copy)]
pub(in crate::set_disk::ops) struct BitrotSelfVerifyTarget<'a> {
pub(in crate::set_disk::ops) operation: &'static str,
pub(in crate::set_disk::ops) bucket: &'a str,
pub(in crate::set_disk::ops) object: &'a str,
pub(in crate::set_disk::ops) part_number: Option<usize>,
pub(in crate::set_disk::ops) volume: &'a str,
pub(in crate::set_disk::ops) path: &'a str,
pub(in crate::set_disk::ops) logical_shard_size: usize,
pub(in crate::set_disk::ops) shard_size: usize,
pub(in crate::set_disk::ops) write_quorum: usize,
}
pub(in crate::set_disk::ops) async fn verify_written_bitrot_shards(
disks: &[Option<DiskStore>],
inline_parts: Option<&[FileInfo]>,
target: BitrotSelfVerifyTarget<'_>,
) -> Result<usize> {
let algo = HashAlgorithm::HighwayHash256S;
let encoded_shard_size = coding::bitrot_shard_file_size(target.logical_shard_size, target.shard_size, algo.clone());
let results = if let Some(parts) = inline_parts {
let tasks = disks.iter().enumerate().filter_map(|(index, disk)| {
let disk = disk.as_ref()?.clone();
let data = parts.get(index).and_then(|part| part.data.as_ref()).cloned();
let algo = algo.clone();
Some(async move {
let result = match data {
Some(data) => coding::bitrot_verify(
Cursor::new(data),
encoded_shard_size,
target.logical_shard_size,
algo,
target.shard_size,
)
.await
.map_err(Error::from),
None => Err(Error::other(format!("{} inline shard {index} is missing bitrot data", target.operation))),
};
(index, disk, result)
})
});
join_all(tasks).await
} else {
let tasks = disks.iter().enumerate().filter_map(|(index, disk)| {
let disk = disk.as_ref()?.clone();
let algo = algo.clone();
Some(async move {
let result = match disk.read_file(target.volume, target.path).await {
Ok(reader) => {
coding::bitrot_verify(reader, encoded_shard_size, target.logical_shard_size, algo, target.shard_size)
.await
.map_err(Error::from)
}
Err(err) => Err(Error::from(err)),
};
(index, disk, result)
})
});
join_all(tasks).await
};
let mut verified_shards = 0usize;
for (index, disk, result) in results {
if let Err(err) = result {
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
operation = target.operation,
bucket = target.bucket,
object = target.object,
part_number = target.part_number,
shard_index = index,
disk = ?disk,
logical_shard_size = target.logical_shard_size,
encoded_shard_size,
error = ?err,
"Set disk no-parity bitrot self-verify failed"
);
return Err(Error::other(format!(
"{} no-parity bitrot self-verify failed for shard {index}: {err}",
target.operation
)));
}
verified_shards += 1;
}
if verified_shards < target.write_quorum {
return Err(Error::other(format!(
"{} no-parity bitrot self-verify quorum unavailable: {verified_shards} shard(s) verified, need {}",
target.operation, target.write_quorum
)));
}
Ok(verified_shards)
}
#[cfg(test)]
mod tests {
use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
use super::*;
use crate::storage_api_contracts::object::ObjectIO as _;
async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes {
let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S);
for chunk in data.chunks(shard_size) {
writer.write(chunk).await.expect("streaming shard should encode");
}
Bytes::from(writer.into_inner().into_inner())
}
#[test]
fn excludes_failed_writers_and_counts_committed() {
let mut disks = vec![Some(0u8), Some(1), Some(2), Some(3)];
let writers = vec![Some(()), Some(()), None, Some(())];
assert_eq!(drop_failed_writer_disks(&mut disks, &writers), 3);
assert_eq!(disks, vec![Some(0), Some(1), None, Some(3)]);
}
#[test]
fn offline_disk_stays_excluded_and_uncounted() {
let mut disks = vec![Some(0u8), None, Some(2)];
let writers = vec![Some(()), None, Some(())];
assert_eq!(drop_failed_writer_disks(&mut disks, &writers), 2);
assert_eq!(disks, vec![Some(0), None, Some(2)]);
}
#[test]
fn all_writers_ok_keeps_every_disk() {
let mut disks = vec![Some(0u8), Some(1), Some(2)];
let writers = vec![Some(()), Some(()), Some(())];
assert_eq!(drop_failed_writer_disks(&mut disks, &writers), 3);
assert_eq!(disks, vec![Some(0), Some(1), Some(2)]);
}
#[tokio::test]
async fn no_parity_self_verify_rejects_issue_5173_final_block_mismatch() {
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
let disk = disks[0].clone();
let shard_size = 1_048_576usize;
let logical_shard_size = 8_250_370usize;
let payload = vec![0x5a; logical_shard_size];
let encoded = encode_streaming_shard(&payload, shard_size).await;
let path = "issue-5173/part.1";
assert_eq!(encoded.len(), 8_250_626);
disk.write_all(RUSTFS_META_TMP_BUCKET, path, encoded.clone())
.await
.expect("healthy temp shard should be written");
let verified = verify_written_bitrot_shards(
&[Some(disk.clone())],
None,
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path,
logical_shard_size,
shard_size,
write_quorum: 1,
},
)
.await
.expect("healthy temp shard should verify");
assert_eq!(verified, 1);
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized))
.await
.expect("oversized temp shard should be staged");
let err = verify_written_bitrot_shards(
&[Some(disk.clone())],
None,
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path,
logical_shard_size,
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("disk shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
let mut corrupt = encoded.to_vec();
let last = corrupt.len() - 1;
corrupt[last] ^= 0x01;
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(corrupt))
.await
.expect("corrupt temp shard should be staged");
let err = verify_written_bitrot_shards(
&[Some(disk)],
None,
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path,
logical_shard_size,
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("corrupt no-parity temp shard must not be committed");
assert!(err.to_string().contains("bitrot self-verify failed"));
}
#[tokio::test]
async fn no_parity_inline_self_verify_rejects_trailing_bytes() {
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
let shard_size = 16usize;
let payload = b"inline bitrot payload";
let encoded = encode_streaming_shard(payload, shard_size).await;
let disks = disks.into_iter().map(Some).collect::<Vec<_>>();
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
let parts = [FileInfo {
data: Some(Bytes::from(oversized)),
..Default::default()
}];
let err = verify_written_bitrot_shards(
&disks,
Some(&parts),
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "inline-object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path: "unused-for-inline",
logical_shard_size: payload.len(),
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("inline shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
}
#[tokio::test]
async fn no_parity_put_round_trip_large_stream() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
let bucket = "no-parity-put-round-trip";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let payload = vec![0x5a; 13 * 1024 * 1024 + 727_213];
let mut reader = crate::set_disk::PutObjReader::from_vec(payload);
set_disks
.put_object(bucket, "object", &mut reader, &crate::set_disk::ObjectOptions::default())
.await
.expect("single-disk no-parity PUT should pass self-verify");
}
}