use std::collections::HashMap;
use std::io::Cursor;
#[cfg(feature = "test-util")]
use std::path::Path;
use std::sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::time::Duration;
use async_trait::async_trait;
use tokio::io::AsyncReadExt;
use tokio::sync::{Mutex, Notify, RwLock};
use uuid::Uuid;
#[cfg(feature = "test-util")]
use crate::disk::endpoint::Endpoint;
#[cfg(feature = "test-util")]
use crate::disk::format::FormatV3;
#[cfg(feature = "test-util")]
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
use crate::services::tier::tier::TierConfigMgr;
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
use crate::services::tier::warm_backend::{
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
#[cfg(feature = "test-util")]
use rustfs_filemeta::FileMeta;
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
#[cfg(feature = "test-util")]
use rustfs_utils::path::path_join_buf;
#[cfg(feature = "test-util")]
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
#[cfg(feature = "test-util")]
impl TransitionCleanupStoreBarrier {
pub fn install() -> Self {
Self(crate::set_disk::SetDiskTransitionCleanupStoreBarrier::install())
}
pub async fn wait_until_paused(&self) {
self.0.wait_until_paused().await;
}
}
#[cfg(feature = "test-util")]
const POLL_INTERVAL: Duration = Duration::from_millis(50);
#[derive(Clone, Debug, Default)]
pub struct FaultConfig {
pub unreachable: bool,
pub server_error: bool,
pub reject_credentials: bool,
pub latency: Option<Duration>,
}
impl FaultConfig {
fn error(&self) -> Option<std::io::Error> {
use std::io::{Error, ErrorKind};
if self.unreachable {
return Some(Error::new(ErrorKind::ConnectionRefused, "mock warm backend unreachable"));
}
if self.reject_credentials {
return Some(Error::new(ErrorKind::PermissionDenied, "mock warm backend rejected credentials"));
}
if self.server_error {
return Some(Error::other("mock warm backend returned HTTP 503"));
}
None
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum MockWarmOp {
Put { object: String },
Get { object: String },
Remove { object: String },
Probe { object: String },
ExternalRemove { object: String },
InUse,
}
#[derive(Clone, Default)]
pub struct MockStoredObject {
pub bytes: Vec<u8>,
pub metadata: HashMap<String, String>,
pub remote_version_id: String,
}
#[derive(Default)]
struct MockWarmBackendInner {
objects: Mutex<HashMap<String, MockStoredObject>>,
faults: Mutex<FaultConfig>,
put_read_limit: Mutex<Option<usize>>,
put_remote_version: Mutex<Option<String>>,
response_loss_after_put: AtomicBool,
transition_candidate_probe_override: Mutex<Option<TransitionCandidateProbe>>,
reject_non_empty_remote_versions: AtomicBool,
reject_non_empty_remote_version_validations: AtomicUsize,
fail_remove: AtomicBool,
exact_remove_count: AtomicUsize,
op_log: Mutex<Vec<MockWarmOp>>,
put_versions: Mutex<Vec<(String, String)>>,
remove_versions: Mutex<Vec<(String, String)>>,
put_barrier: Mutex<Option<Arc<MockPutBarrierState>>>,
get_barrier: Mutex<Option<Arc<MockGetBarrierState>>>,
remove_barrier: Mutex<Option<Arc<MockRemoveBarrierState>>>,
}
#[derive(Default)]
struct MockPutBarrierState {
arrived: Notify,
release: Notify,
}
#[derive(Default)]
struct MockGetBarrierState {
arrived: Notify,
release: Notify,
fail_after_release: bool,
}
#[derive(Default)]
struct MockRemoveBarrierState {
arrived: Notify,
release: Notify,
operation_dropped: Notify,
}
struct MockRemoveOperationGuard {
state: Arc<MockRemoveBarrierState>,
}
impl Drop for MockRemoveOperationGuard {
fn drop(&mut self) {
self.state.operation_dropped.notify_one();
}
}
#[cfg(feature = "test-util")]
pub struct MockPutBarrier {
state: Arc<MockPutBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockPutBarrier {
pub async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("mock tier PUT should reach the deterministic barrier");
}
pub fn release(&self) {
self.state.release.notify_one();
}
}
#[cfg(feature = "test-util")]
impl Drop for MockPutBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
}
}
pub struct MockGetBarrier {
state: Arc<MockGetBarrierState>,
}
impl MockGetBarrier {
pub async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("mock tier GET should reach the deterministic barrier");
}
pub fn release(&self) {
self.state.release.notify_one();
}
}
impl Drop for MockGetBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
}
}
#[cfg(feature = "test-util")]
pub struct MockRemoveBarrier {
state: Arc<MockRemoveBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockRemoveBarrier {
pub async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("mock tier DELETE should reach the deterministic barrier");
}
pub fn release(&self) {
self.state.release.notify_one();
}
pub async fn wait_until_operation_dropped(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.operation_dropped.notified())
.await
.expect("mock tier DELETE operation should be dropped");
}
}
#[cfg(feature = "test-util")]
impl Drop for MockRemoveBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
}
}
#[derive(Clone, Default)]
pub struct MockWarmBackend {
inner: Arc<MockWarmBackendInner>,
}
impl MockWarmBackend {
pub fn new() -> Self {
Self::default()
}
#[cfg(feature = "test-util")]
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
let state = Arc::new(MockPutBarrierState::default());
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
MockPutBarrier { state }
}
#[cfg(feature = "test-util")]
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
let state = Arc::new(MockRemoveBarrierState::default());
let mut barrier = self.inner.remove_barrier.lock().await;
assert!(barrier.is_none(), "mock tier DELETE barrier is already armed");
*barrier = Some(state.clone());
MockRemoveBarrier { state }
}
#[cfg(feature = "test-util")]
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
let state = Arc::new(MockGetBarrierState {
fail_after_release: true,
..Default::default()
});
*self.inner.get_barrier.lock().await = Some(Arc::clone(&state));
MockGetBarrier { state }
}
pub async fn arm_get_barrier(&self) -> MockGetBarrier {
let state = Arc::new(MockGetBarrierState::default());
*self.inner.get_barrier.lock().await = Some(Arc::clone(&state));
MockGetBarrier { state }
}
#[cfg(feature = "test-util")]
pub async fn set_faults(&self, faults: FaultConfig) {
*self.inner.faults.lock().await = faults;
}
pub async fn set_unreachable(&self, unreachable: bool) {
self.inner.faults.lock().await.unreachable = unreachable;
}
#[cfg(feature = "test-util")]
pub async fn set_server_error(&self, server_error: bool) {
self.inner.faults.lock().await.server_error = server_error;
}
pub async fn set_reject_credentials(&self, reject: bool) {
self.inner.faults.lock().await.reject_credentials = reject;
}
#[cfg(feature = "test-util")]
pub async fn set_latency(&self, latency: Option<Duration>) {
self.inner.faults.lock().await.latency = latency;
}
#[cfg(feature = "test-util")]
pub async fn clear_faults(&self) {
*self.inner.faults.lock().await = FaultConfig::default();
}
#[cfg(feature = "test-util")]
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
*self.inner.put_read_limit.lock().await = limit;
}
pub async fn set_put_remote_version(&self, remote_version: Option<String>) {
*self.inner.put_remote_version.lock().await = remote_version;
}
pub fn lose_next_put_response(&self) {
self.inner.response_loss_after_put.store(true, Ordering::Release);
}
pub async fn set_transition_candidate_probe_override(&self, probe: Option<TransitionCandidateProbe>) {
*self.inner.transition_candidate_probe_override.lock().await = probe;
}
#[cfg(feature = "test-util")]
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
}
#[cfg(feature = "test-util")]
pub fn reject_next_non_empty_remote_version_validation(&self) {
self.inner
.reject_non_empty_remote_version_validations
.fetch_add(1, Ordering::AcqRel);
}
pub fn set_remove_failure(&self, fail: bool) {
self.inner.fail_remove.store(fail, Ordering::Release);
}
async fn precondition(&self) -> Result<(), std::io::Error> {
let (latency, error) = {
let faults = self.inner.faults.lock().await;
(faults.latency, faults.error())
};
if let Some(latency) = latency {
tokio::time::sleep(latency).await;
}
match error {
Some(err) => Err(err),
None => Ok(()),
}
}
async fn record(&self, op: MockWarmOp) {
self.inner.op_log.lock().await.push(op);
}
pub async fn op_log(&self) -> Vec<MockWarmOp> {
self.inner.op_log.lock().await.clone()
}
#[cfg(feature = "test-util")]
pub async fn clear_op_log(&self) {
self.inner.op_log.lock().await.clear();
}
pub async fn remove_count(&self) -> usize {
self.inner
.op_log
.lock()
.await
.iter()
.filter(|op| matches!(op, MockWarmOp::Remove { .. }))
.count()
}
pub fn exact_remove_count(&self) -> usize {
self.inner.exact_remove_count.load(Ordering::Acquire)
}
#[cfg(feature = "test-util")]
pub async fn put_versions(&self) -> Vec<(String, String)> {
self.inner.put_versions.lock().await.clone()
}
#[cfg(feature = "test-util")]
pub async fn remove_versions(&self) -> Vec<(String, String)> {
self.inner.remove_versions.lock().await.clone()
}
pub async fn probe_transition_candidate_state(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probe_transition_candidate(object).await
}
#[cfg(feature = "test-util")]
pub async fn get_count(&self) -> usize {
self.inner
.op_log
.lock()
.await
.iter()
.filter(|op| matches!(op, MockWarmOp::Get { .. }))
.count()
}
#[cfg(feature = "test-util")]
pub async fn put_count(&self) -> usize {
self.inner
.op_log
.lock()
.await
.iter()
.filter(|op| matches!(op, MockWarmOp::Put { .. }))
.count()
}
#[cfg(feature = "test-util")]
pub async fn contains(&self, object: &str) -> bool {
self.inner.objects.lock().await.contains_key(object)
}
pub async fn object_count(&self) -> usize {
self.inner.objects.lock().await.len()
}
#[cfg(feature = "test-util")]
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
self.inner.objects.lock().await.get(object).cloned()
}
#[cfg(feature = "test-util")]
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
}
pub async fn metadata(&self, object: &str) -> Option<HashMap<String, String>> {
self.inner.objects.lock().await.get(object).map(|o| o.metadata.clone())
}
pub async fn external_remove(&self, object: &str) {
self.inner.objects.lock().await.remove(object);
self.record(MockWarmOp::ExternalRemove {
object: object.to_string(),
})
.await;
}
#[cfg(feature = "test-util")]
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if !self.contains(object).await {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(POLL_INTERVAL).await;
}
}
#[cfg(feature = "test-util")]
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if self.object_count().await == expected {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(POLL_INTERVAL).await;
}
}
async fn put_bytes(&self, object: &str, bytes: Vec<u8>, metadata: HashMap<String, String>) -> String {
let remote_version_id = self
.inner
.put_remote_version
.lock()
.await
.clone()
.unwrap_or_else(|| Uuid::new_v4().to_string());
self.inner.objects.lock().await.insert(
object.to_string(),
MockStoredObject {
bytes,
metadata,
remote_version_id: remote_version_id.clone(),
},
);
remote_version_id
}
async fn read_bytes(&self, reader: ReaderImpl) -> Result<Vec<u8>, std::io::Error> {
let limit = *self.inner.put_read_limit.lock().await;
match reader {
ReaderImpl::Body(bytes) => Ok(bytes.slice(..limit.unwrap_or(bytes.len()).min(bytes.len())).to_vec()),
ReaderImpl::ObjectBody(mut reader) => {
let mut buf = Vec::new();
if let Some(limit) = limit {
let limit =
u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?;
(&mut reader).take(limit).read_to_end(&mut buf).await?;
} else {
reader.read_to_end(&mut buf).await?;
}
Ok(buf)
}
}
}
fn maybe_lose_put_response(&self) -> Result<(), std::io::Error> {
if self.inner.response_loss_after_put.swap(false, Ordering::AcqRel) {
return Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"mock warm backend lost PUT response after storing remote object",
));
}
Ok(())
}
}
#[async_trait]
impl WarmBackend for MockWarmBackend {
async fn probe_legacy_metadata(
&self,
object: &str,
remote_version: Option<&str>,
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
use super::warm_backend::LegacyTransitionStateProbe as Probe;
let candidate = match remote_version {
Some(version) if !version.is_empty() => self.probe_transition_version(object, version).await?,
_ => self.probe_transition_candidate(object).await?,
};
Ok(match candidate {
TransitionCandidateProbe::Missing => Probe::Missing,
TransitionCandidateProbe::UnversionedPresent => Probe::UnversionedPresent,
TransitionCandidateProbe::VersionedPresent(version) if version == "null" => Probe::SuspendedNullPresent,
TransitionCandidateProbe::VersionedPresent(version) => Probe::VersionedPresent(version),
TransitionCandidateProbe::Ambiguous => Probe::Ambiguous,
TransitionCandidateProbe::Unsupported => Probe::Unsupported,
})
}
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
if remote_version_id.is_empty() {
return Ok(());
}
let reject_once = self
.inner
.reject_non_empty_remote_version_validations
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
.is_ok();
if reject_once || self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) {
return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));
}
Ok(())
}
async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
self.precondition().await?;
let bytes = self.read_bytes(r).await?;
let version = self.put_bytes(object, bytes, HashMap::new()).await;
self.inner
.put_versions
.lock()
.await
.push((object.to_string(), version.clone()));
self.record(MockWarmOp::Put {
object: object.to_string(),
})
.await;
self.maybe_lose_put_response()?;
Ok(version)
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
_length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
self.precondition().await?;
let bytes = self.read_bytes(r).await?;
let opts = build_transition_put_options(String::new(), meta);
let mut metadata = opts.user_metadata.clone();
if !opts.content_type.is_empty() {
metadata.insert("content-type".to_string(), opts.content_type.clone());
}
if !opts.content_encoding.is_empty() {
metadata.insert("content-encoding".to_string(), opts.content_encoding.clone());
}
if !opts.cache_control.is_empty() {
metadata.insert("cache-control".to_string(), opts.cache_control.clone());
}
if !opts.internal.replication_status.as_str().is_empty() {
metadata.insert(
"x-amz-replication-status".to_string(),
opts.internal.replication_status.as_str().to_string(),
);
}
if !opts.legalhold.as_str().is_empty() {
metadata.insert("x-amz-object-lock-legal-hold".to_string(), opts.legalhold.as_str().to_string());
}
let version = self.put_bytes(object, bytes, metadata).await;
self.inner
.put_versions
.lock()
.await
.push((object.to_string(), version.clone()));
let barrier = self.inner.put_barrier.lock().await.take();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
self.record(MockWarmOp::Put {
object: object.to_string(),
})
.await;
self.maybe_lose_put_response()?;
Ok(version)
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.precondition().await?;
let barrier = self.inner.get_barrier.lock().await.take();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
if barrier.fail_after_release {
return Err(std::io::Error::other("mock warm backend GET failed after barrier"));
}
}
self.record(MockWarmOp::Get {
object: object.to_string(),
})
.await;
let objects = self.inner.objects.lock().await;
let Some(stored) = objects.get(object) else {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
};
if !rv.is_empty() && stored.remote_version_id != rv {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
}
let bytes = &stored.bytes;
let start = opts.start_offset.max(0) as usize;
let end = if opts.length > 0 {
start.saturating_add(opts.length as usize).min(bytes.len())
} else {
bytes.len()
};
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.precondition().await?;
if let Some(barrier) = self.inner.remove_barrier.lock().await.take() {
let _operation = MockRemoveOperationGuard { state: barrier.clone() };
barrier.arrived.notify_one();
barrier.release.notified().await;
return Err(std::io::Error::other("mock warm backend remove failure after barrier"));
}
if self.inner.fail_remove.load(Ordering::Acquire) {
return Err(std::io::Error::other("mock warm backend remove failure"));
}
let mut objects = self.inner.objects.lock().await;
if let Some(stored) = objects.get(object)
&& !rv.is_empty()
&& stored.remote_version_id != rv
{
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
}
objects.remove(object);
drop(objects);
self.inner
.remove_versions
.lock()
.await
.push((object.to_string(), rv.to_string()));
self.record(MockWarmOp::Remove {
object: object.to_string(),
})
.await;
Ok(())
}
async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
if rv.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"an exact mock tier delete requires a remote version ID",
));
}
self.inner.exact_remove_count.fetch_add(1, Ordering::AcqRel);
self.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.precondition().await?;
self.record(MockWarmOp::Probe {
object: object.to_string(),
})
.await;
if let Some(probe) = self.inner.transition_candidate_probe_override.lock().await.clone() {
return Ok(probe);
}
let objects = self.inner.objects.lock().await;
let Some(stored) = objects.get(object) else {
return Ok(TransitionCandidateProbe::Missing);
};
if stored.remote_version_id.is_empty() {
Ok(TransitionCandidateProbe::UnversionedPresent)
} else {
Ok(TransitionCandidateProbe::VersionedPresent(stored.remote_version_id.clone()))
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.precondition().await?;
self.record(MockWarmOp::InUse).await;
Ok(!self.inner.objects.lock().await.is_empty())
}
}
pub async fn register_mock_tier(handle: &Arc<RwLock<TierConfigMgr>>, tier_name: &str) -> MockWarmBackend {
let backend = MockWarmBackend::new();
register_mock_tier_backend(handle, tier_name, backend.clone()).await;
backend
}
pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tier_name: &str, backend: MockWarmBackend) {
let mut tier_config_mgr = handle.write().await;
tier_config_mgr.tiers.insert(
tier_name.to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::MinIO,
name: tier_name.to_string(),
minio: Some(TierMinIO {
access_key: "minioadmin".to_string(),
secret_key: "minioadmin".to_string(),
bucket: "mock-tier".to_string(),
endpoint: "http://127.0.0.1:0".to_string(),
prefix: format!("mock/{}/", Uuid::new_v4()),
region: String::new(),
..Default::default()
}),
..Default::default()
},
);
drop(tier_config_mgr);
TierConfigMgr::install_test_driver_in(handle, tier_name, Box::new(backend))
.await
.expect("mock tier driver should install");
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg(feature = "test-util")]
pub struct TransitionMeta {
pub status: String,
pub tier: String,
pub remote_object: String,
pub remote_version_id: Option<String>,
pub free_version_count: usize,
}
#[cfg(feature = "test-util")]
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
let format_data = tokio::fs::read(disk_path.join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE))
.await
.ok()?;
let format = FormatV3::try_from(format_data.as_slice()).ok()?;
let (set_idx, disk_idx) = format.find_disk_index_by_disk_id(format.erasure.this).ok()?;
let mut endpoint = Endpoint::try_from(disk_path.to_str()?).ok()?;
endpoint.set_pool_index(0);
endpoint.set_set_index(set_idx);
endpoint.set_disk_index(disk_idx);
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.ok()
}
#[cfg(feature = "test-util")]
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
let Some(disk) = open_disk(disk_path).await else {
return 0;
};
let Ok(data) = disk
.read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
.await
else {
return 0;
};
let Ok(meta) = FileMeta::load(&data) else {
return 0;
};
meta.get_file_info_versions(bucket, object, false)
.map(|v| v.free_versions.len())
.unwrap_or(0)
}
#[cfg(feature = "test-util")]
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
let disk = open_disk(disk_path).await?;
let data = disk
.read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
.await
.ok()?;
let meta = FileMeta::load(&data).ok()?;
let versions = meta.get_file_info_versions(bucket, object, false).ok()?;
let free_version_count = versions.free_versions.len();
let fi = versions
.versions
.iter()
.find(|v| !v.transition_status.is_empty() || !v.transitioned_objname.is_empty())
.or_else(|| versions.versions.first())?;
Some(TransitionMeta {
status: fi.transition_status.clone(),
tier: fi.transition_tier.clone(),
remote_object: fi.transitioned_objname.clone(),
remote_version_id: fi
.transition_version
.clone()
.or_else(|| fi.transition_version_id.map(|id| id.to_string())),
free_version_count,
})
}
#[cfg(feature = "test-util")]
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
let mut expected: Option<TransitionMeta> = None;
for disk_path in disk_paths {
let disk_path = disk_path.as_ref();
let meta = read_transition_meta(disk_path, bucket, object)
.await
.unwrap_or_else(|| panic!("missing xl.meta for {bucket}/{object} on disk {}", disk_path.display()));
match &expected {
None => expected = Some(meta),
Some(expected) => assert_eq!(
*expected,
meta,
"transition metadata mismatch for {bucket}/{object} on disk {} (expected {expected:?}, got {meta:?})",
disk_path.display()
),
}
}
expected.expect("at least one disk was inspected")
}
#[cfg(feature = "test-util")]
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if free_version_count(disk_path, bucket, object).await == 0 {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(POLL_INTERVAL).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use bytes::Bytes;
#[tokio::test]
async fn mock_probe_distinguishes_missing_unversioned_and_versioned_candidates() {
let backend = MockWarmBackend::new();
assert_eq!(
backend
.probe_transition_candidate_state("missing")
.await
.expect("probe missing candidate"),
TransitionCandidateProbe::Missing
);
backend.set_put_remote_version(Some(String::new())).await;
backend
.put("unversioned", ReaderImpl::Body(Bytes::new()), 0)
.await
.expect("put unversioned candidate");
assert_eq!(
backend
.probe_transition_candidate_state("unversioned")
.await
.expect("probe unversioned candidate"),
TransitionCandidateProbe::UnversionedPresent
);
let remote_version = Uuid::new_v4().to_string();
backend.set_put_remote_version(Some(remote_version.clone())).await;
backend
.put("versioned", ReaderImpl::Body(Bytes::new()), 0)
.await
.expect("put versioned candidate");
assert_eq!(
backend
.probe_transition_candidate_state("versioned")
.await
.expect("probe versioned candidate"),
TransitionCandidateProbe::VersionedPresent(remote_version)
);
assert_eq!(
backend
.op_log()
.await
.into_iter()
.filter(|op| matches!(op, MockWarmOp::Probe { .. }))
.count(),
3
);
}
#[tokio::test]
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
let backend = MockWarmBackend::new();
let metadata = HashMap::from([
("content-type".to_string(), "text/plain".to_string()),
("project".to_string(), "archive".to_string()),
]);
let version = backend
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
.await
.expect("mock PUT should preserve remote metadata");
assert_eq!(backend.metadata("object").await, Some(metadata));
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe stored object"),
TransitionCandidateProbe::VersionedPresent(version)
);
backend.external_remove("object").await;
assert_eq!(backend.metadata("object").await, None);
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe removed object"),
TransitionCandidateProbe::Missing
);
let operations = backend.op_log().await;
assert!(
operations
.iter()
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
);
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
}
#[tokio::test]
async fn mock_probe_preserves_fault_fail_closed_behavior() {
let backend = MockWarmBackend::new();
backend.set_reject_credentials(true).await;
let err = backend
.probe_transition_candidate("remote-object")
.await
.expect_err("credential rejection must fail the authoritative probe");
assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
assert!(backend.op_log().await.is_empty());
}
}