use super::super::{
Arc, CancellationToken, DiskError, ListPathRawOptions, MetaCacheEntries, MetaCacheEntry, SetDisks, debug, disk, list_path_raw,
};
#[cfg(test)]
use crate::disk::DiskAPI;
use crate::object_api::ObjectInfo;
#[cfg(test)]
use rustfs_filemeta::FileMeta;
use std::collections::HashSet;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60);
#[derive(Debug, Clone)]
pub struct HealWalkVersion {
pub name: String,
pub version_id: Option<String>,
pub mod_time_unix_nanos: Option<i128>,
pub lifecycle_object_info: Option<ObjectInfo>,
pub is_delete_marker: bool,
}
#[derive(Debug, Clone)]
struct HealWalkObject {
name: String,
versions: Vec<HealWalkVersion>,
}
struct HealWalkCollector {
bucket: String,
prefix: String,
forward_to: Option<String>,
batch_objects: usize,
version_budget: usize,
include_lifecycle_object_info: bool,
objects: Mutex<Vec<HealWalkObject>>,
decode_error: Mutex<Option<DiskError>>,
version_total: AtomicUsize,
truncated: AtomicBool,
cancel: CancellationToken,
}
impl HealWalkCollector {
fn accepts_name(&self, name: &str) -> bool {
name.starts_with(&self.prefix) && self.forward_to.as_deref().is_none_or(|forward_to| name >= forward_to)
}
fn lock_objects(&self) -> disk::error::Result<std::sync::MutexGuard<'_, Vec<HealWalkObject>>> {
self.objects.lock().map_err(|_| {
self.cancel.cancel();
DiskError::FileCorrupt
})
}
fn record_decode_error(&self, error: rustfs_filemeta::Error) {
if let Ok(mut first_error) = self.decode_error.lock()
&& first_error.is_none()
{
*first_error = Some(error.into());
}
self.cancel.cancel();
}
fn record_object(&self, name: String, versions: Vec<HealWalkVersion>) -> Option<(usize, usize)> {
let mut objects = self.lock_objects().ok()?;
let mut added = 0;
if let Some(existing) = objects.iter_mut().find(|object| object.name == name) {
let mut seen: HashSet<String> = existing
.versions
.iter()
.filter_map(|version| version.version_id.clone())
.collect();
for version in versions {
if seen.insert(version.version_id.clone().unwrap_or_default()) {
existing.versions.push(version);
added += 1;
}
}
} else {
added = versions.len();
objects.push(HealWalkObject { name, versions });
}
let ver_total = self.version_total.fetch_add(added, Ordering::SeqCst) + added;
Some((objects.len(), ver_total))
}
fn take_decode_error(&self) -> disk::error::Result<Option<DiskError>> {
self.decode_error.lock().map(|mut error| error.take()).map_err(|_| {
self.cancel.cancel();
DiskError::FileCorrupt
})
}
fn ingest(&self, entry: MetaCacheEntry) {
if !self.accepts_name(&entry.name) {
return;
}
if entry.is_dir() {
return;
}
let fiv = match entry.file_info_versions_with_free_versions(&self.bucket) {
Ok(fiv) => fiv,
Err(err) => {
self.record_decode_error(err);
return;
}
};
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
let lifecycle_object_info = if self.include_lifecycle_object_info {
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
} else {
None
};
versions.push(HealWalkVersion {
name: entry.name.clone(),
version_id: Some(fi.version_id.unwrap_or_default().to_string()),
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
lifecycle_object_info,
is_delete_marker: fi.deleted,
});
}
if versions.is_empty() {
return;
}
let Some((objs_len, ver_total)) = self.record_object(entry.name, versions) else {
return;
};
if objs_len >= self.batch_objects || ver_total >= self.version_budget {
self.truncated.store(true, Ordering::SeqCst);
self.cancel.cancel();
}
}
fn ingest_merged(&self, entries: &MetaCacheEntries) {
let mut name = String::new();
let mut seen = HashSet::new();
let mut versions = Vec::new();
for entry in entries.0.iter().flatten() {
if !self.accepts_name(&entry.name) || entry.is_dir() || entry.name.is_empty() {
continue;
}
if name.is_empty() {
name = entry.name.clone();
}
let fiv = match entry.file_info_versions_with_free_versions(&self.bucket) {
Ok(fiv) => fiv,
Err(err) => {
debug!(entry = %entry.name, error = ?err, "heal disk-walk merged skipped entry with unreadable versions");
continue;
}
};
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
let vid = Some(fi.version_id.unwrap_or_default().to_string());
if seen.insert(vid.clone()) {
let lifecycle_object_info = if self.include_lifecycle_object_info {
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
} else {
None
};
versions.push(HealWalkVersion {
name: entry.name.clone(),
version_id: vid,
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
lifecycle_object_info,
is_delete_marker: fi.deleted,
});
}
}
}
if versions.is_empty() || name.is_empty() {
return;
}
let Some((objs_len, ver_total)) = self.record_object(name, versions) else {
return;
};
if objs_len >= self.batch_objects || ver_total >= self.version_budget {
self.truncated.store(true, Ordering::SeqCst);
self.cancel.cancel();
}
}
}
fn finalize_heal_walk_page(
mut objects: Vec<HealWalkObject>,
forward_to: Option<&str>,
truncated: bool,
) -> (Vec<HealWalkVersion>, Option<String>, bool) {
let next_forward = objects.last().map(|o| o.name.clone());
if let Some(fw) = forward_to {
while objects.first().map(|o| o.name.as_str()) == Some(fw) {
objects.remove(0);
}
}
let versions: Vec<HealWalkVersion> = objects.into_iter().flat_map(|o| o.versions).collect();
if truncated {
(versions, next_forward, true)
} else {
(versions, None, false)
}
}
impl SetDisks {
pub(crate) async fn heal_walk_versions_page(
&self,
bucket: &str,
prefix: &str,
forward_to: Option<&str>,
batch_objects: usize,
version_budget: usize,
include_lifecycle_object_info: bool,
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
let disks = self.get_disks_internal().await;
let collector = Arc::new(HealWalkCollector {
bucket: bucket.to_string(),
prefix: prefix.to_string(),
forward_to: forward_to.map(str::to_owned),
batch_objects,
version_budget: version_budget.max(1),
include_lifecycle_object_info,
objects: Mutex::new(Vec::new()),
decode_error: Mutex::new(None),
version_total: AtomicUsize::new(0),
truncated: AtomicBool::new(false),
cancel: CancellationToken::new(),
});
let agreed_collector = collector.clone();
let partial_collector = collector.clone();
let path = rustfs_utils::path::base_dir_from_prefix(prefix);
let filter_prefix = if prefix.is_empty() {
None
} else {
Some(
prefix
.trim_start_matches(&path)
.trim_start_matches('/')
.trim_end_matches('/')
.to_owned(),
)
};
let opts = ListPathRawOptions {
disks,
bucket: bucket.to_string(),
path,
recursive: true,
incl_deleted: true,
filter_prefix,
forward_to: forward_to.map(str::to_string),
min_disks: 1,
report_not_found: false,
per_disk_limit: 0,
skip_walkdir_total_timeout: true,
walkdir_stall_timeout: Some(BACKGROUND_WALKDIR_STALL_TIMEOUT),
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
let collector = agreed_collector.clone();
Box::pin(async move {
collector.ingest(entry);
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _errs: &[Option<DiskError>]| {
let collector = partial_collector.clone();
Box::pin(async move {
collector.ingest_merged(&entries);
})
})),
finished: None,
..Default::default()
};
let walk_result = list_path_raw(collector.cancel.clone(), opts).await;
if let Some(err) = collector.take_decode_error()? {
return Err(err);
}
match walk_result {
Ok(()) => {}
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {
debug!(bucket, prefix, "heal disk-walk treated missing path as empty page");
}
Err(err) => return Err(err),
}
let objects = {
let mut objects = collector.lock_objects()?;
std::mem::take(&mut *objects)
};
let truncated = collector.truncated.load(Ordering::SeqCst);
Ok(finalize_heal_walk_page(objects, forward_to, truncated))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMetaVersion, MetaObject, VersionType};
use time::OffsetDateTime;
use uuid::Uuid;
fn test_collector() -> Arc<HealWalkCollector> {
test_collector_for("", None)
}
fn test_collector_for(prefix: &str, forward_to: Option<&str>) -> Arc<HealWalkCollector> {
Arc::new(HealWalkCollector {
bucket: "bucket".to_string(),
prefix: prefix.to_string(),
forward_to: forward_to.map(str::to_owned),
batch_objects: 2,
version_budget: 2,
include_lifecycle_object_info: false,
objects: Mutex::new(Vec::new()),
decode_error: Mutex::new(None),
version_total: AtomicUsize::new(0),
truncated: AtomicBool::new(false),
cancel: CancellationToken::new(),
})
}
#[test]
fn collector_rechecks_full_prefix_and_forward_cursor() {
let collector = test_collector_for("a/b/", Some("a/b/child"));
assert!(!collector.accepts_name("a/"), "base marker outside the requested prefix must be ignored");
assert!(!collector.accepts_name("a/b-other/object"), "sibling prefixes must not be included");
assert!(
!collector.accepts_name("a/b/before"),
"objects before the inclusive cursor must be ignored"
);
assert!(collector.accepts_name("a/b/child"), "the cursor boundary is inclusive");
assert!(collector.accepts_name("a/b/child/deep"), "descendants after the cursor must be included");
}
#[tokio::test]
async fn heal_walk_scopes_non_root_prefix_to_nested_objects() {
use rustfs_filemeta::FileInfo;
let (temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(4).await;
let bucket = "nested-prefix-bucket";
for disk in &disks {
disk.make_volume(bucket).await.expect("test bucket should be created");
}
let objects = [
"a/",
"a/b",
"a/b/direct",
"a/b/child/deep",
"a/b/child/later",
"a/b-other/sibling",
"a/c/outside",
"single",
"中文/%2F+ key/item",
];
for (index, object) in objects.iter().enumerate() {
let mut info = FileInfo::new(object, 4, 2);
info.volume = bucket.to_string();
info.name = object.to_string();
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("index fits UUID")));
info.versioned = true;
info.size = if object.ends_with('/') { 0 } else { 100 };
info.mod_time = Some(
OffsetDateTime::from_unix_timestamp(i64::try_from(index + 1).expect("index fits timestamp"))
.expect("fixture timestamp"),
);
let mut metadata = FileMeta::new();
metadata
.add_version(info)
.expect("fixture metadata should accept the version");
let object_dir = temp_dirs[0]
.path()
.join(bucket)
.join(rustfs_utils::path::encode_dir_object(object));
tokio::fs::create_dir_all(&object_dir)
.await
.expect("object directory should be created");
tokio::fs::write(
object_dir.join(crate::disk::STORAGE_FORMAT_FILE),
metadata.marshal_msg().expect("fixture metadata should serialize"),
)
.await
.expect("fixture metadata should be written");
}
for prefix in ["", "a/", "a/b", "a/b/", "single", "中文/", "中文/%2F+", "absent/"] {
let mut expected: Vec<_> = objects.iter().copied().filter(|object| object.starts_with(prefix)).collect();
expected.sort_unstable();
let mut actual = Vec::new();
let mut forward: Option<String> = None;
let mut pages = 0;
loop {
let (versions, next, truncated) = set_disks
.heal_walk_versions_page(bucket, prefix, forward.as_deref(), 2, 100, false)
.await
.expect("prefix disk walk should succeed");
pages += 1;
actual.extend(versions.into_iter().map(|version| version.name));
if !truncated {
assert!(next.is_none(), "complete prefix page must clear its cursor");
break;
}
let next = next.expect("truncated prefix page must return a cursor");
assert!(next.starts_with(prefix), "cursor must retain the full logical key");
assert!(forward.as_ref().is_none_or(|previous| &next > previous), "cursor must advance");
forward = Some(next);
assert!(pages < 20, "prefix pagination must terminate");
}
actual.sort_unstable();
assert_eq!(actual, expected, "prefix {prefix:?} must enumerate every matching key exactly once");
}
}
#[test]
fn collectors_preserve_historical_null_identity() {
use rustfs_filemeta::FileInfo;
let latest = Uuid::from_u128(7);
for null_marker in [false, true] {
let mut metadata = FileMeta::new();
for (version, seconds, deleted) in [(Uuid::nil(), 1, null_marker), (latest, 2, false)] {
let mut info = if deleted {
FileInfo::default()
} else {
FileInfo::new("object", 4, 2)
};
info.name = "object".to_string();
info.volume = "bucket".to_string();
info.version_id = Some(version);
info.versioned = true;
info.deleted = deleted;
info.size = if deleted { 0 } else { 100 };
info.mod_time = Some(OffsetDateTime::from_unix_timestamp(seconds).expect("fixture timestamp"));
metadata.add_version(info).expect("fixture version should be valid");
}
let entry = MetaCacheEntry {
name: "object".to_string(),
metadata: metadata.marshal_msg().expect("fixture metadata should serialize"),
..Default::default()
};
for merged in [false, true] {
let collector = test_collector();
if merged {
collector.ingest_merged(&MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone())]));
} else {
collector.ingest(entry.clone());
}
let objects = collector.lock_objects().expect("collector should remain readable");
assert_eq!(objects.len(), 1);
let versions = &objects[0].versions;
assert_eq!(versions.len(), 2, "one exact unit per version, including merged duplicates");
let null = versions
.iter()
.find(|item| item.version_id.as_deref() == Some(Uuid::nil().to_string().as_str()))
.expect("historical null must remain an explicit selector");
assert_eq!(null.is_delete_marker, null_marker);
assert!(
versions
.iter()
.any(|item| item.version_id.as_deref() == Some(latest.to_string().as_str()))
);
}
}
}
fn crc_valid_semantically_corrupt_entry(name: &str) -> MetaCacheEntry {
let mut metadata = FileMeta::new();
metadata
.add_version_filemata(FileMetaVersion {
version_type: VersionType::Object,
object: Some(MetaObject {
version_id: Some(Uuid::new_v4()),
erasure_algorithm: ErasureAlgo::ReedSolomon,
erasure_m: 2,
erasure_n: 2,
erasure_block_size: 1 << 20,
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
part_numbers: vec![1, 2],
part_sizes: vec![10],
part_actual_sizes: vec![10, 20],
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
}),
..Default::default()
})
.expect("corrupt test version should be accepted before semantic decoding");
MetaCacheEntry {
name: name.to_string(),
metadata: metadata.marshal_msg().expect("test metadata should encode with a valid CRC"),
..Default::default()
}
}
fn version(name: &str, id: &str, dm: bool) -> HealWalkVersion {
HealWalkVersion {
name: name.to_string(),
version_id: Some(id.to_string()),
mod_time_unix_nanos: None,
lifecycle_object_info: None,
is_delete_marker: dm,
}
}
fn object_with_versions(name: &str, count: usize) -> HealWalkObject {
HealWalkObject {
name: name.to_string(),
versions: (0..count).map(|i| version(name, &format!("{name}-v{i}"), false)).collect(),
}
}
#[test]
fn union_page_bounded_by_version_budget_on_heavily_versioned_object() {
let big = object_with_versions("big.bin", 50);
let (versions, next_forward, truncated) = finalize_heal_walk_page(vec![big], None, true);
assert_eq!(versions.len(), 50, "the heavily-versioned object must not be split across pages");
assert!(truncated, "exceeding the version budget must mark the page truncated");
assert_eq!(
next_forward.as_deref(),
Some("big.bin"),
"resume cursor must point at the boundary object"
);
}
#[test]
fn union_page_de_overlaps_inclusive_forward_boundary() {
let a = object_with_versions("a.bin", 2);
let b = object_with_versions("b.bin", 3);
let (versions, next_forward, truncated) = finalize_heal_walk_page(vec![a, b], Some("a.bin"), true);
assert!(
versions.iter().all(|v| v.name == "b.bin"),
"the boundary object a.bin must be de-overlapped, got {versions:?}"
);
assert_eq!(versions.len(), 3);
assert_eq!(next_forward.as_deref(), Some("b.bin"));
assert!(truncated);
}
#[test]
fn union_page_non_truncated_clears_cursor() {
let a = object_with_versions("a.bin", 1);
let (versions, next_forward, truncated) = finalize_heal_walk_page(vec![a], None, false);
assert_eq!(versions.len(), 1);
assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor");
assert!(!truncated);
}
#[test]
fn ingest_merged_surfaces_versions_from_divergent_entries() {
use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntries, MetaCacheEntry};
use time::OffsetDateTime;
use uuid::Uuid;
let t0 = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let t1 = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let make_entry = |name: &str, version_u128: u128, deleted: bool, ts: OffsetDateTime| -> MetaCacheEntry {
let mut fi = FileInfo::new(name, 4, 2);
fi.volume = "bucket".to_string();
fi.name = name.to_string();
fi.version_id = Some(Uuid::from_u128(version_u128));
fi.versioned = true;
fi.deleted = deleted;
fi.size = if deleted { 0 } else { 100 };
fi.mod_time = Some(ts);
fi.metadata = [("etag".to_string(), format!("etag-{version_u128:x}"))].into();
let mut meta = FileMeta::new();
meta.add_version(fi).expect("test metadata should accept version");
let encoded = meta.marshal_msg().expect("test metadata should marshal");
MetaCacheEntry {
name: name.to_string(),
metadata: encoded,
cached: Some(meta),
reusable: false,
}
};
let stale_entry = make_entry("a.txt", 0xBEEF, false, t0); let delete_entry = make_entry("a.txt", 0xCAFE, true, t1);
let collector = Arc::new(HealWalkCollector {
bucket: "bucket".to_string(),
prefix: "".to_string(),
forward_to: None,
batch_objects: 1000,
version_budget: 10_000,
include_lifecycle_object_info: false,
objects: Mutex::new(Vec::new()),
version_total: AtomicUsize::new(0),
decode_error: Mutex::new(None),
truncated: AtomicBool::new(false),
cancel: CancellationToken::new(),
});
let entries = MetaCacheEntries(vec![Some(stale_entry), Some(delete_entry.clone()), Some(delete_entry)]);
collector.ingest_merged(&entries);
let objects = collector.lock_objects().expect("mutex should not be poisoned");
assert_eq!(objects.len(), 1, "both entries share the same object name");
let versions = &objects[0].versions;
let version_ids: std::collections::HashSet<Option<String>> = versions.iter().map(|v| v.version_id.clone()).collect();
assert!(
version_ids.contains(&Some("00000000-0000-0000-0000-00000000beef".to_string())),
"ingest_merged must surface the stale data version from the returning node: {version_ids:?}"
);
assert!(
version_ids.contains(&Some("00000000-0000-0000-0000-00000000cafe".to_string())),
"ingest_merged must also surface the delete marker from the quorum: {version_ids:?}"
);
assert_eq!(versions.len(), 2, "exactly two unique versions must be collected");
}
#[test]
fn poisoned_collector_state_cancels_the_walk() {
let collector = test_collector();
let poison_target = Arc::clone(&collector);
let _ = std::thread::spawn(move || {
let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
panic!("poison heal collector");
})
.join();
let error = collector.lock_objects().expect_err("poisoned collection must fail closed");
assert_eq!(error, DiskError::FileCorrupt);
assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
}
#[test]
fn semantic_decode_failure_records_error_and_cancels_walk() {
let collector = test_collector();
let entry = crc_valid_semantically_corrupt_entry("corrupt-object");
collector.ingest(entry);
let error = collector
.take_decode_error()
.expect("decode error state should remain readable")
.expect("semantic metadata corruption must be recorded");
assert_eq!(error, DiskError::FileCorrupt);
assert!(collector.cancel.is_cancelled(), "semantic metadata corruption must cancel the disk walk");
}
#[tokio::test]
async fn heal_walk_returns_crc_valid_semantic_decode_failure() {
let bucket = "bucket";
let object = "corrupt-object";
let (temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(1).await;
disks[0].make_volume(bucket).await.expect("test bucket should be created");
let object_dir = temp_dirs[0].path().join(bucket).join(object);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("test object directory should be created");
tokio::fs::write(
object_dir.join(crate::disk::STORAGE_FORMAT_FILE),
crc_valid_semantically_corrupt_entry(object).metadata,
)
.await
.expect("corrupt test metadata should be written");
let error = set_disks
.heal_walk_versions_page(bucket, "", None, 2, 2, false)
.await
.expect_err("semantic metadata corruption must fail the heal disk walk");
assert_eq!(error, DiskError::FileCorrupt);
}
}