use super::global::TypeLocalDiskSetDrives;
use crate::bucket::bandwidth::monitor::Monitor;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState};
use crate::bucket::metadata_sys::BucketMetadataSys;
use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
use crate::disk::DiskStore;
use crate::layout::endpoints::{EndpointServerPools, SetupType};
use crate::object_api::ObjectEncryptionResolver;
use crate::services::event_notification::EventNotifier;
use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) expires_at: Instant,
pub(crate) movement_generation: u64,
pub(crate) namespace_generation: u64,
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
pub(crate) struct NamespaceCommitGuard {
ctx: Arc<InstanceContext>,
counted: bool,
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
self.ctx.advance_namespace_commit_generation();
self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
}
}
}
pub struct InstanceContext {
erasure_kind: RwLock<SetupType>,
lock_manager: Arc<GlobalLockManager>,
region: OnceLock<Region>,
deployment_id: OnceLock<Uuid>,
endpoints: OnceLock<EndpointServerPools>,
tier_config_mgr: OnceLock<Arc<RwLock<TierConfigMgr>>>,
event_notifier: OnceLock<Arc<RwLock<EventNotifier>>>,
expiry_state: OnceLock<Arc<RwLock<ExpiryState>>>,
transition_state: OnceLock<Arc<TransitionState>>,
bucket_monitor: OnceLock<Arc<Monitor>>,
replication_stats: OnceCell<Arc<ReplicationStats>>,
replication_pool: OnceCell<Arc<DynReplicationPool>>,
local_disk_map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
local_disk_id_map: Arc<RwLock<HashMap<Uuid, String>>>,
local_disk_set_drives: Arc<RwLock<TypeLocalDiskSetDrives>>,
bucket_metadata_sys: std::sync::Mutex<Option<Arc<RwLock<BucketMetadataSys>>>>,
background_cancel_token: OnceLock<CancellationToken>,
data_movement_operation_gate: Arc<RwLock<()>>,
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
data_movement_operation_epoch: AtomicU64,
data_movement_operation_epoch_exhausted: AtomicBool,
data_movement_generation: AtomicU64,
data_movement_generation_exhausted: AtomicBool,
data_movement_generation_notify: Arc<Notify>,
scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
suppress_tier_delete_journal_recovery: bool,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
}
impl InstanceContext {
pub fn new() -> Self {
Self::with_lock_manager(Arc::new(GlobalLockManager::new()))
}
fn with_lock_manager(lock_manager: Arc<GlobalLockManager>) -> Self {
Self {
erasure_kind: RwLock::new(SetupType::Unknown),
lock_manager,
region: OnceLock::new(),
deployment_id: OnceLock::new(),
endpoints: OnceLock::new(),
tier_config_mgr: OnceLock::new(),
event_notifier: OnceLock::new(),
expiry_state: OnceLock::new(),
transition_state: OnceLock::new(),
bucket_monitor: OnceLock::new(),
replication_stats: OnceCell::new(),
replication_pool: OnceCell::new(),
local_disk_map: Arc::new(RwLock::new(HashMap::new())),
local_disk_id_map: Arc::new(RwLock::new(HashMap::new())),
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
data_movement_operation_gate: Arc::new(RwLock::new(())),
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
data_movement_generation: AtomicU64::new(0),
data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
suppress_tier_delete_journal_recovery: false,
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
}
}
#[cfg(test)]
pub(crate) fn with_lock_manager_for_test(lock_manager: Arc<GlobalLockManager>) -> Self {
Self::with_lock_manager(lock_manager)
}
pub fn lock_manager(&self) -> Arc<GlobalLockManager> {
self.lock_manager.clone()
}
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) async fn install_scanner_publication_lease(
&self,
token: Uuid,
expires_at: Instant,
movement_generation: u64,
namespace_generation: u64,
operation_guard: OwnedRwLockReadGuard<()>,
) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
return false;
}
leases.insert(
token,
ScannerPublicationLeaseEntry {
expires_at,
movement_generation,
namespace_generation,
_operation_guard: operation_guard,
},
);
true
}
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
self.scanner_publication_leases.lock().await.remove(&token).is_some()
}
pub(crate) async fn scanner_publication_lease_generations(&self, token: Uuid) -> Option<(u64, u64)> {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let (expires_at, movement_generation, namespace_generation) = leases
.get(&token)
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation))?;
if expires_at <= now {
leases.remove(&token);
return None;
}
Some((movement_generation, namespace_generation))
}
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
let mut leases = self.scanner_publication_leases.lock().await;
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
if should_remove {
leases.remove(&token);
}
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation(&self) -> u64 {
self.data_movement_generation.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
self.data_movement_generation_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
Arc::clone(&self.data_movement_generation_notify)
}
pub(crate) fn observe_durable_data_movement_generation(&self, generation: u64) {
if generation == 0 || self.data_movement_generation_exhausted.load(Ordering::Acquire) {
return;
}
let previous = self.data_movement_generation.fetch_max(generation, Ordering::AcqRel);
if generation == u64::MAX {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
}
if generation > previous {
self.data_movement_generation_notify.notify_waiters();
}
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& !self.data_movement_generation_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
let counted = self
.namespace_commits
.try_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
.is_ok();
if counted {
self.advance_namespace_commit_generation();
} else {
self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
}
Arc::new(NamespaceCommitGuard {
ctx: Arc::clone(self),
counted,
})
}
fn advance_namespace_commit_generation(&self) {
let _ = self
.namespace_commit_generation
.try_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
}
pub(crate) fn namespace_commit_generation(&self) -> u64 {
self.namespace_commit_generation.load(Ordering::Acquire)
}
pub(crate) fn namespace_commits_pending(&self) -> bool {
self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
let (previous, result) = self.advance_data_movement_operation_epoch_only();
if result != previous {
let _ = self.advance_data_movement_generation();
}
result
}
pub(crate) fn advance_data_movement_operation_epoch_to_durable_generation(&self, generation: u64) -> u64 {
let (_, result) = self.advance_data_movement_operation_epoch_only();
self.observe_durable_data_movement_generation(generation);
result
}
fn advance_data_movement_operation_epoch_only(&self) -> (u64, u64) {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
let _ = self
.data_movement_operation_epoch
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
(previous, result)
}
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
return None;
}
let updated = self
.data_movement_generation
.try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
match updated {
Ok(previous) => {
let Some(generation) = previous.checked_add(1) else {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
return None;
};
if generation == u64::MAX {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
}
self.data_movement_generation_notify.notify_waiters();
Some(generation)
}
Err(_) => {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
None
}
}
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_namespace_commit_generation_for_test(&self, generation: u64) {
self.namespace_commit_generation.store(generation, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
self.data_movement_generation.store(generation, Ordering::Release);
self.data_movement_generation_exhausted
.store(generation == u64::MAX, Ordering::Release);
}
pub fn set_object_encryption_resolver(
&self,
resolver: Arc<dyn ObjectEncryptionResolver>,
) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
self.object_encryption_resolver.set(resolver)
}
pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
self.object_encryption_resolver.get().map(Arc::as_ref)
}
pub fn set_region(&self, region: Region) {
self.region
.set(region)
.expect("instance region should be initialized once during startup");
}
pub fn region(&self) -> Option<Region> {
self.region.get().cloned()
}
pub fn set_deployment_id(&self, id: Uuid) {
self.deployment_id
.set(id)
.expect("instance deployment id should be initialized once during startup");
}
pub fn deployment_id(&self) -> Option<Uuid> {
self.deployment_id.get().copied()
}
pub fn set_endpoints(&self, endpoints: EndpointServerPools) {
self.endpoints
.set(endpoints)
.expect("instance endpoints should be initialized once during storage startup");
}
pub fn endpoints(&self) -> Option<EndpointServerPools> {
self.endpoints.get().cloned()
}
pub fn tier_config_mgr(&self) -> Arc<RwLock<TierConfigMgr>> {
self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
}
pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
self.event_notifier.get_or_init(EventNotifier::new).clone()
}
pub fn expiry_state(&self) -> Arc<RwLock<ExpiryState>> {
self.expiry_state.get_or_init(ExpiryState::new).clone()
}
pub fn transition_state(&self) -> Arc<TransitionState> {
self.transition_state.get_or_init(TransitionState::new).clone()
}
pub fn init_bucket_monitor(&self, num_nodes: u64) -> bool {
self.bucket_monitor.set(Monitor::new(num_nodes)).is_ok()
}
pub fn bucket_monitor(&self) -> Option<Arc<Monitor>> {
self.bucket_monitor.get().cloned()
}
pub fn replication_stats(&self) -> Option<Arc<ReplicationStats>> {
self.replication_stats.get().cloned()
}
pub fn replication_pool(&self) -> Option<Arc<DynReplicationPool>> {
self.replication_pool.get().cloned()
}
pub fn replication_initialized(&self) -> bool {
self.replication_stats.get().is_some() && self.replication_pool.get().is_some()
}
pub(crate) fn replication_stats_cell(&self) -> &OnceCell<Arc<ReplicationStats>> {
&self.replication_stats
}
pub(crate) fn replication_pool_cell(&self) -> &OnceCell<Arc<DynReplicationPool>> {
&self.replication_pool
}
pub(crate) fn local_disk_map(&self) -> Arc<RwLock<HashMap<String, Option<DiskStore>>>> {
self.local_disk_map.clone()
}
pub(crate) fn local_disk_id_map(&self) -> Arc<RwLock<HashMap<Uuid, String>>> {
self.local_disk_id_map.clone()
}
pub(crate) fn local_disk_set_drives(&self) -> Arc<RwLock<TypeLocalDiskSetDrives>> {
self.local_disk_set_drives.clone()
}
pub fn init_bucket_metadata_sys(&self, sys: Arc<RwLock<BucketMetadataSys>>) -> bool {
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
let is_first = guard.is_none();
*guard = Some(sys);
is_first
}
pub fn bucket_metadata_sys(&self) -> Option<Arc<RwLock<BucketMetadataSys>>> {
self.bucket_metadata_sys
.lock()
.expect("bucket_metadata_sys lock poisoned")
.clone()
}
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
self.background_cancel_token.set(token)
}
pub fn background_cancel_token(&self) -> Option<CancellationToken> {
self.background_cancel_token.get().cloned()
}
pub(crate) fn bind_background_cancel_token(&self, token: CancellationToken) -> CancellationToken {
self.background_cancel_token.get_or_init(|| token).clone()
}
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
#[cfg(test)]
if self.suppress_tier_delete_journal_recovery {
return false;
}
self.tier_delete_journal_recovery_stores
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(store_id)
}
#[cfg(test)]
pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
self.suppress_tier_delete_journal_recovery = true;
}
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
self.transition_transaction_recovery_stores
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(store_id)
}
pub(crate) fn wake_tier_delete_journal_recovery(&self) {
self.tier_delete_journal_recovery_wakeup.notify_one();
}
pub(crate) async fn wait_for_tier_delete_journal_recovery(&self) {
self.tier_delete_journal_recovery_wakeup.notified().await;
}
pub async fn update_erasure_type(&self, setup_type: SetupType) {
*self.erasure_kind.write().await = setup_type;
}
pub async fn is_erasure(&self) -> bool {
matches!(*self.erasure_kind.read().await, SetupType::Erasure | SetupType::DistErasure)
}
pub async fn is_dist_erasure(&self) -> bool {
*self.erasure_kind.read().await == SetupType::DistErasure
}
pub async fn is_erasure_sd(&self) -> bool {
*self.erasure_kind.read().await == SetupType::ErasureSD
}
}
impl Default for InstanceContext {
fn default() -> Self {
Self::new()
}
}
impl std::fmt::Debug for InstanceContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("InstanceContext")
.field("erasure_kind", &self.erasure_kind)
.field("region", &self.region)
.field("deployment_id", &self.deployment_id)
.field("endpoints", &self.endpoints)
.field("tier_config_mgr_set", &self.tier_config_mgr.get().is_some())
.field("event_notifier_set", &self.event_notifier.get().is_some())
.field("expiry_state_set", &self.expiry_state.get().is_some())
.field("transition_state_set", &self.transition_state.get().is_some())
.field("bucket_monitor_set", &self.bucket_monitor.get().is_some())
.field(
"bucket_metadata_sys_set",
&self.bucket_metadata_sys.lock().map(|g| g.is_some()).unwrap_or(false),
)
.field("replication_stats_set", &self.replication_stats.get().is_some())
.field("replication_pool_set", &self.replication_pool.get().is_some())
.field("background_cancel_token_set", &self.background_cancel_token.get().is_some())
.field(
"tier_delete_journal_recovery_store_count",
&self
.tier_delete_journal_recovery_stores
.lock()
.map_or(0, |stores| stores.len()),
)
.field(
"transition_transaction_recovery_store_count",
&self
.transition_transaction_recovery_stores
.lock()
.map_or(0, |stores| stores.len()),
)
.finish_non_exhaustive()
}
}
static BOOTSTRAP_CTX: OnceLock<Arc<InstanceContext>> = OnceLock::new();
pub fn bootstrap_ctx() -> Arc<InstanceContext> {
BOOTSTRAP_CTX
.get_or_init(|| Arc::new(InstanceContext::with_lock_manager(get_global_lock_manager())))
.clone()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
let first = Arc::new(InstanceContext::new());
let other = Arc::new(InstanceContext::new());
first.set_scanner_publication_state(false);
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
assert_eq!(first.namespace_commit_generation(), 2);
assert!(!other.namespace_commits_pending());
assert_eq!(other.namespace_commit_generation(), 0);
assert!(other.scanner_publication_state_allowed());
drop(one);
assert_eq!(first.namespace_commit_generation(), 2);
drop(shared_owner);
assert!(first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 3);
drop(two);
assert!(!first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 4);
assert!(first.scanner_publication_state_allowed());
}
#[test]
fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
let ctx = Arc::new(InstanceContext::new());
ctx.set_scanner_publication_state(false);
ctx.namespace_commits.store(count, Ordering::Release);
ctx.namespace_commit_generation.store(generation, Ordering::Release);
let guard = ctx.begin_namespace_commit();
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
drop(guard);
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
}
}
#[tokio::test(start_paused = true)]
async fn scanner_lease_generations_remain_bound_until_expiry() {
let ctx = Arc::new(InstanceContext::new());
let token = Uuid::new_v4();
let gate = ctx.data_movement_operation_gate();
let permit = gate.clone().read_owned().await;
assert!(
ctx.install_scanner_publication_lease(token, Instant::now() + SCANNER_PUBLICATION_LEASE_TTL, 7, 11, permit)
.await
);
drop(ctx.begin_namespace_commit());
assert_eq!(ctx.namespace_commit_generation(), 2);
assert_eq!(ctx.scanner_publication_lease_generations(token).await, Some((7, 11)));
assert!(gate.clone().try_write_owned().is_err(), "lookup must retain the stored permit");
tokio::time::advance(SCANNER_PUBLICATION_LEASE_TTL).await;
assert_eq!(ctx.scanner_publication_lease_generations(token).await, None);
assert!(!ctx.remove_scanner_publication_lease(token).await);
assert!(gate.try_write_owned().is_ok(), "expiry releases the stored permit");
}
#[tokio::test]
async fn erasure_predicates_match_legacy_derivation() {
let cases = [
(SetupType::Unknown, false, false, false),
(SetupType::FS, false, false, false),
(SetupType::Erasure, true, false, false),
(SetupType::DistErasure, true, true, false),
(SetupType::ErasureSD, false, false, true),
];
for (input, want_erasure, want_dist, want_sd) in cases {
let ctx = InstanceContext::new();
ctx.update_erasure_type(input.clone()).await;
assert_eq!(ctx.is_erasure().await, want_erasure, "is_erasure for {input:?}");
assert_eq!(ctx.is_dist_erasure().await, want_dist, "is_dist_erasure for {input:?}");
assert_eq!(ctx.is_erasure_sd().await, want_sd, "is_erasure_sd for {input:?}");
}
}
#[tokio::test]
async fn fresh_context_is_all_false() {
let ctx = InstanceContext::new();
assert!(!ctx.is_erasure().await);
assert!(!ctx.is_dist_erasure().await);
assert!(!ctx.is_erasure_sd().await);
}
#[tokio::test]
async fn bootstrap_ctx_is_stable_singleton() {
let a = bootstrap_ctx();
let b = bootstrap_ctx();
assert!(Arc::ptr_eq(&a, &b), "bootstrap_ctx must return the same Arc");
}
#[tokio::test]
async fn distinct_contexts_do_not_share_state() {
let ctx_a = Arc::new(InstanceContext::new());
let ctx_b = Arc::new(InstanceContext::new());
ctx_a.update_erasure_type(SetupType::DistErasure).await;
ctx_b.update_erasure_type(SetupType::ErasureSD).await;
assert!(ctx_a.is_dist_erasure().await && ctx_a.is_erasure().await);
assert!(!ctx_a.is_erasure_sd().await);
assert!(ctx_b.is_erasure_sd().await);
assert!(!ctx_b.is_erasure().await && !ctx_b.is_dist_erasure().await);
}
#[tokio::test]
async fn bootstrap_lock_manager_aliases_process_singleton() {
assert!(
Arc::ptr_eq(&bootstrap_ctx().lock_manager(), &get_global_lock_manager()),
"bootstrap context must alias the process lock-manager singleton"
);
}
#[tokio::test]
async fn distinct_contexts_have_distinct_lock_managers() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
assert!(
!Arc::ptr_eq(&ctx_a.lock_manager(), &ctx_b.lock_manager()),
"fresh contexts must not share a lock manager"
);
}
fn region(name: &str) -> Region {
name.parse().expect("valid test region")
}
#[tokio::test]
async fn region_set_and_get() {
let ctx = InstanceContext::new();
assert_eq!(ctx.region(), None);
ctx.set_region(region("us-east-1"));
assert_eq!(ctx.region(), Some(region("us-east-1")));
}
#[tokio::test]
async fn distinct_contexts_have_distinct_region() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
ctx_a.set_region(region("us-east-1"));
ctx_b.set_region(region("eu-west-1"));
assert_eq!(ctx_a.region(), Some(region("us-east-1")));
assert_eq!(ctx_b.region(), Some(region("eu-west-1")));
}
#[tokio::test]
#[should_panic(expected = "initialized once")]
async fn set_region_twice_panics() {
let ctx = InstanceContext::new();
ctx.set_region(region("us-east-1"));
ctx.set_region(region("eu-west-1"));
}
#[tokio::test]
async fn deployment_id_set_and_get() {
let ctx = InstanceContext::new();
assert_eq!(ctx.deployment_id(), None);
let id = Uuid::new_v4();
ctx.set_deployment_id(id);
assert_eq!(ctx.deployment_id(), Some(id));
}
#[tokio::test]
async fn distinct_contexts_have_distinct_deployment_id() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
let id_a = Uuid::new_v4();
let id_b = Uuid::new_v4();
ctx_a.set_deployment_id(id_a);
ctx_b.set_deployment_id(id_b);
assert_eq!(ctx_a.deployment_id(), Some(id_a));
assert_eq!(ctx_b.deployment_id(), Some(id_b));
}
#[tokio::test]
#[should_panic(expected = "initialized once")]
async fn set_deployment_id_twice_panics() {
let ctx = InstanceContext::new();
ctx.set_deployment_id(Uuid::new_v4());
ctx.set_deployment_id(Uuid::new_v4());
}
fn endpoints_with_pools(n: usize) -> EndpointServerPools {
use crate::layout::endpoint::Endpoint;
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
let pool = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: Endpoints::from(vec![Endpoint::try_from("http://127.0.0.1:9000/data0").expect("valid endpoint")]),
cmd_line: "test".to_string(),
platform: "test".to_string(),
};
EndpointServerPools((0..n).map(|_| pool.clone()).collect())
}
#[tokio::test]
async fn endpoints_set_and_get() {
let ctx = InstanceContext::new();
assert!(ctx.endpoints().is_none());
ctx.set_endpoints(endpoints_with_pools(1));
assert_eq!(ctx.endpoints().expect("endpoints set").0.len(), 1);
}
#[tokio::test]
async fn distinct_contexts_have_distinct_endpoints() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
ctx_a.set_endpoints(endpoints_with_pools(1));
ctx_b.set_endpoints(endpoints_with_pools(2));
assert_eq!(ctx_a.endpoints().expect("a").0.len(), 1);
assert_eq!(ctx_b.endpoints().expect("b").0.len(), 2);
}
#[tokio::test]
#[should_panic(expected = "initialized once")]
async fn set_endpoints_twice_panics() {
let ctx = InstanceContext::new();
ctx.set_endpoints(endpoints_with_pools(1));
ctx.set_endpoints(endpoints_with_pools(2));
}
#[tokio::test]
async fn tier_config_mgr_is_stable_within_an_instance() {
let ctx = InstanceContext::new();
assert!(
Arc::ptr_eq(&ctx.tier_config_mgr(), &ctx.tier_config_mgr()),
"repeated access must return the same manager"
);
}
#[tokio::test]
async fn distinct_contexts_have_distinct_tier_config_mgr() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
assert!(
!Arc::ptr_eq(&ctx_a.tier_config_mgr(), &ctx_b.tier_config_mgr()),
"fresh contexts must not share a tier config manager"
);
}
#[tokio::test]
async fn event_notifier_is_stable_within_an_instance() {
let ctx = InstanceContext::new();
assert!(
Arc::ptr_eq(&ctx.event_notifier(), &ctx.event_notifier()),
"repeated access must return the same notifier"
);
}
#[tokio::test]
async fn distinct_contexts_have_distinct_event_notifier() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
assert!(
!Arc::ptr_eq(&ctx_a.event_notifier(), &ctx_b.event_notifier()),
"fresh contexts must not share an event notifier"
);
}
#[tokio::test]
async fn lifecycle_state_is_stable_within_and_distinct_across_instances() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
assert!(Arc::ptr_eq(&ctx_a.expiry_state(), &ctx_a.expiry_state()));
assert!(Arc::ptr_eq(&ctx_a.transition_state(), &ctx_a.transition_state()));
assert!(!Arc::ptr_eq(&ctx_a.expiry_state(), &ctx_b.expiry_state()));
assert!(!Arc::ptr_eq(&ctx_a.transition_state(), &ctx_b.transition_state()));
}
#[tokio::test]
async fn bucket_monitor_init_once_and_isolated() {
let ctx_a = InstanceContext::new();
assert!(ctx_a.bucket_monitor().is_none());
assert!(ctx_a.init_bucket_monitor(4));
assert!(ctx_a.bucket_monitor().is_some());
assert!(!ctx_a.init_bucket_monitor(8), "second init must be ignored");
let ctx_b = InstanceContext::new();
assert!(ctx_b.bucket_monitor().is_none(), "a fresh instance has its own uninitialized monitor");
}
#[tokio::test]
async fn replication_state_is_isolated() {
let ctx_a = InstanceContext::new();
assert!(ctx_a.replication_stats().is_none());
assert!(ctx_a.replication_pool().is_none());
assert!(!ctx_a.replication_initialized());
ctx_a.replication_stats_cell().set(Arc::new(ReplicationStats::new())).ok();
assert!(ctx_a.replication_stats().is_some());
assert!(!ctx_a.replication_initialized());
let ctx_b = InstanceContext::new();
assert!(
ctx_b.replication_stats().is_none(),
"a distinct instance has independent replication state"
);
}
#[tokio::test]
async fn background_cancel_token_init_once_and_isolated() {
let ctx_a = InstanceContext::new();
assert!(ctx_a.background_cancel_token().is_none());
let token = CancellationToken::new();
ctx_a.init_background_cancel_token(token.clone()).expect("init once");
assert!(ctx_a.background_cancel_token().is_some());
assert!(
ctx_a.init_background_cancel_token(CancellationToken::new()).is_err(),
"second init rejected"
);
let ctx_b = InstanceContext::new();
assert!(ctx_b.background_cancel_token().is_none(), "distinct instance is independent");
token.cancel();
assert!(ctx_a.background_cancel_token().unwrap().is_cancelled());
}
#[test]
fn tier_delete_journal_recovery_is_deduplicated_per_store_and_instance() {
let ctx_a = InstanceContext::new();
let ctx_b = InstanceContext::new();
let store_a = Uuid::new_v4();
let store_b = Uuid::new_v4();
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_a));
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
let mut manual_ctx = InstanceContext::new();
manual_ctx.suppress_tier_delete_journal_recovery_for_test();
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
}
#[test]
fn background_cancel_token_binds_the_provided_shutdown_token() {
let ctx = InstanceContext::new();
let shutdown = CancellationToken::new();
let first = ctx.bind_background_cancel_token(shutdown.clone());
let second = ctx.bind_background_cancel_token(CancellationToken::new());
shutdown.cancel();
assert!(first.is_cancelled());
assert!(second.is_cancelled());
assert!(
ctx.background_cancel_token()
.expect("shutdown token should be published")
.is_cancelled()
);
}
#[tokio::test]
async fn two_instances_isolate_all_migrated_state() {
let a = InstanceContext::new();
let b = InstanceContext::new();
a.update_erasure_type(SetupType::DistErasure).await;
b.update_erasure_type(SetupType::ErasureSD).await;
assert!(a.is_dist_erasure().await && !b.is_dist_erasure().await);
assert!(b.is_erasure_sd().await && !a.is_erasure_sd().await);
assert!(!Arc::ptr_eq(&a.lock_manager(), &b.lock_manager()));
a.set_region("us-east-1".parse().expect("region"));
b.set_region("eu-west-1".parse().expect("region"));
assert_eq!(a.region().expect("a region"), "us-east-1".parse().expect("region"));
assert_eq!(b.region().expect("b region"), "eu-west-1".parse().expect("region"));
let (id_a, id_b) = (Uuid::new_v4(), Uuid::new_v4());
a.set_deployment_id(id_a);
b.set_deployment_id(id_b);
assert_eq!(a.deployment_id(), Some(id_a));
assert_eq!(b.deployment_id(), Some(id_b));
assert!(!Arc::ptr_eq(&a.tier_config_mgr(), &b.tier_config_mgr()));
assert!(!Arc::ptr_eq(&a.event_notifier(), &b.event_notifier()));
assert!(!Arc::ptr_eq(&a.expiry_state(), &b.expiry_state()));
assert!(!Arc::ptr_eq(&a.transition_state(), &b.transition_state()));
a.local_disk_map().write().await.insert("/disk-a".to_string(), None);
assert_eq!(a.local_disk_map().read().await.len(), 1);
assert_eq!(b.local_disk_map().read().await.len(), 0);
assert!(a.init_bucket_monitor(4));
assert!(a.bucket_monitor().is_some() && b.bucket_monitor().is_none());
let token_a = CancellationToken::new();
a.init_background_cancel_token(token_a.clone()).expect("init a");
token_a.cancel();
assert!(a.background_cancel_token().expect("a token").is_cancelled());
assert!(b.background_cancel_token().is_none());
}
}