use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Weak};
use parking_lot::{Condvar, Mutex, MutexGuard, RwLock};
use serde::{Deserialize, Serialize};
use super::org::{OrgError, OrgId, OrgRevocationBundle};
use crate::adapter::net::identity::EntityId;
pub const ORG_REVOCATION_STATE_VERSION: u32 = 1;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OrgRevocationState {
floors: BTreeMap<(OrgId, EntityId), u32>,
}
impl OrgRevocationState {
pub fn empty() -> Self {
Self::default()
}
#[cfg(test)]
pub(crate) fn from_floors_for_test(floors: BTreeMap<(OrgId, EntityId), u32>) -> Self {
Self { floors }
}
pub fn floor_for(&self, org: &OrgId, member: &EntityId) -> u32 {
self.floors
.get(&(*org, member.clone()))
.copied()
.unwrap_or(0)
}
pub fn len(&self) -> usize {
self.floors.len()
}
pub fn is_empty(&self) -> bool {
self.floors.is_empty()
}
pub fn iter(&self) -> impl Iterator<Item = (&(OrgId, EntityId), &u32)> {
self.floors.iter()
}
pub fn merge_bundle(&mut self, bundle: &OrgRevocationBundle) -> usize {
let mut raised = 0;
for (member, floor) in bundle.floors() {
if *floor == 0 {
continue;
}
let entry = self
.floors
.entry((bundle.org_id, member.clone()))
.or_insert(0);
if *floor > *entry {
*entry = *floor;
raised += 1;
}
}
raised
}
fn to_file_bytes(&self) -> Result<Vec<u8>, OrgRevocationError> {
let file = PersistedStateFile {
version: ORG_REVOCATION_STATE_VERSION,
floors: self
.floors
.iter()
.map(|((org, member), floor)| PersistedFloor {
org: *org,
member: member.clone(),
floor: *floor,
})
.collect(),
};
serde_json::to_vec_pretty(&file).map_err(|e| OrgRevocationError::Io {
path: String::new(),
reason: format!("serialize revocation state: {e}"),
})
}
pub fn load_if_exists(path: &Path) -> Result<Option<Self>, OrgRevocationError> {
match read_regular_nofollow(path) {
Ok(bytes) => Self::from_file_bytes(&bytes, path).map(Some),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason: e.to_string(),
}),
}
}
fn from_file_bytes(bytes: &[u8], path: &Path) -> Result<Self, OrgRevocationError> {
let file: PersistedStateFile =
serde_json::from_slice(bytes).map_err(|e| OrgRevocationError::CorruptState {
path: path.display().to_string(),
detail: e.to_string(),
})?;
if file.version != ORG_REVOCATION_STATE_VERSION {
return Err(OrgRevocationError::UnsupportedVersion {
path: path.display().to_string(),
found: file.version,
});
}
let mut floors = BTreeMap::new();
for entry in file.floors {
if entry.floor == 0 {
continue;
}
if floors
.insert((entry.org, entry.member), entry.floor)
.is_some()
{
return Err(OrgRevocationError::CorruptState {
path: path.display().to_string(),
detail: "duplicate (org, member) floor entry".to_string(),
});
}
}
if floors.len() >= FLOOR_COUNT_ADVISORY {
tracing::warn!(
floors = floors.len(),
path = %path.display(),
"org revocation: the persisted floor set is large; every raise \
re-serializes it under the interprocess lock and every authority \
install takes one exclusive fold lock per entry. Consider rolling \
the org certificate generation so historical floors can be retired.",
);
}
Ok(Self { floors })
}
}
const FLOOR_COUNT_ADVISORY: usize = 4_096;
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct PersistedStateFile {
version: u32,
floors: Vec<PersistedFloor>,
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct PersistedFloor {
org: OrgId,
member: EntityId,
floor: u32,
}
#[derive(Debug)]
pub enum OrgRevocationError {
InvalidBundle(OrgError),
MissingState {
path: String,
},
CorruptState {
path: String,
detail: String,
},
UnsupportedVersion {
path: String,
found: u32,
},
Io {
path: String,
reason: String,
},
DurabilityUncertain {
path: String,
reason: String,
},
Poisoned {
path: String,
},
NonMonotonicReplacement {
path: String,
},
BackingIdentityConflict {
path: String,
},
}
impl std::fmt::Display for OrgRevocationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::InvalidBundle(e) => write!(f, "revocation bundle rejected: {e}"),
Self::MissingState { path } => write!(
f,
"revocation state file missing at {path}; refusing to start with \
implicitly empty floors — run `net node adopt` to provision"
),
Self::CorruptState { path, detail } => write!(
f,
"revocation state file at {path} is corrupt ({detail}); refusing to \
start against silently weaker floors"
),
Self::UnsupportedVersion { path, found } => write!(
f,
"revocation state file at {path} has unsupported version {found} \
(this build supports {ORG_REVOCATION_STATE_VERSION})"
),
Self::Io { path, reason } => write!(f, "revocation state I/O at {path}: {reason}"),
Self::DurabilityUncertain { path, reason } => write!(
f,
"revocation state at {path}: rename landed but the parent-directory \
fsync failed ({reason}); disk and memory can no longer be proven \
synchronized — path poisoned until a locked reread and a successful \
parent-directory fsync recover it"
),
Self::Poisoned { path } => write!(
f,
"revocation store path {path} is poisoned after a durability-uncertain \
write; recovery requires a locked reread republished through the \
shared store plus a successful parent-directory fsync (restarting the \
process is one route, not the requirement)"
),
Self::NonMonotonicReplacement { path } => write!(
f,
"refusing to replace the installed revocation store with {path}: its \
live view is lower on at least one (org, member) floor — an installed \
floor never lowers; apply a bundle instead"
),
Self::BackingIdentityConflict { path } => write!(
f,
"revocation store path {path} is bound to a different .lock sidecar \
identity than the one just opened — the sidecar was recreated or \
replaced while a same-path core is still live; refusing to fork the \
path into two independent security views"
),
}
}
}
impl std::error::Error for OrgRevocationError {}
pub type RaisedFloor = (OrgId, EntityId, u32);
type FloorsRaisedCallback = Arc<dyn Fn(&[RaisedFloor]) + Send + Sync>;
struct StoreCore {
path: PathBuf,
backing_id: BackingId,
reload: Mutex<()>,
live: RwLock<Arc<OrgRevocationState>>,
generation: AtomicU64,
generation_exhausted: AtomicBool,
poison_gate: Mutex<()>,
#[cfg(test)]
publish_contended_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
#[cfg(test)]
poison_blocking_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
#[cfg(test)]
force_post_rename: AtomicBool,
#[cfg(test)]
poison_contended_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
subscribers: RwLock<Vec<(u64, FloorsRaisedCallback)>>,
next_subscriber: AtomicU64,
#[cfg(any(test, feature = "fixtures"))]
publish_pause: parking_lot::Mutex<Option<PublishPauseHook>>,
}
#[cfg(any(test, feature = "fixtures"))]
struct PublishPauseHook {
swapped: std::sync::mpsc::Sender<()>,
resume: std::sync::mpsc::Receiver<()>,
}
impl StoreCore {
fn publish(&self, mut next: OrgRevocationState) -> Vec<RaisedFloor> {
#[cfg(test)]
let mut live = match self.live.try_write() {
Some(guard) => guard,
None => {
let hook = self.publish_contended_hook.lock().clone();
if let Some(hook) = hook {
hook();
}
self.live.write()
}
};
#[cfg(not(test))]
let mut live = self.live.write();
let mut regressed = 0usize;
for ((org, member), floor) in live.iter() {
if *floor == 0 {
continue;
}
let entry = next.floors.entry((*org, member.clone())).or_insert(*floor);
if *floor > *entry {
*entry = *floor;
regressed += 1;
}
}
if regressed > 0 {
tracing::error!(
keys = regressed,
"org revocation: the persisted state is BEHIND the enforced view \
for {regressed} floor(s); the live view is preserved, but disk \
will re-persist the weaker base and a restart would roll those \
floors back. Restore the state file or re-apply the bundles \
that raised them.",
);
}
let raised: Vec<RaisedFloor> = next
.iter()
.filter(|((org, member), floor)| **floor > live.floor_for(org, member))
.map(|((org, member), floor)| (*org, member.clone(), *floor))
.collect();
*live = Arc::new(next);
#[cfg(any(test, feature = "fixtures"))]
self.run_publish_pause_hook();
let current = self.generation.load(Ordering::Acquire);
match current.checked_add(1) {
Some(next) => self.generation.store(next, Ordering::Release),
None => {
if !self.generation_exhausted.swap(true, Ordering::AcqRel) {
tracing::error!(
"org revocation: publication generation space exhausted; \
the generation is frozen and every generation-based \
currentness check must now fail closed"
);
}
}
}
raised
}
#[cfg(any(test, feature = "fixtures"))]
fn run_publish_pause_hook(&self) {
if let Some(hook) = self.publish_pause.lock().take() {
let _ = hook.swapped.send(());
let _ = hook.resume.recv();
}
}
fn notify(&self, raised: &[RaisedFloor]) {
if raised.is_empty() {
return;
}
let subscribers: Vec<FloorsRaisedCallback> = self
.subscribers
.read()
.iter()
.map(|(_, callback)| callback.clone())
.collect();
for callback in subscribers {
callback(raised);
}
}
fn notify_authority_changed(&self) {
let subscribers: Vec<FloorsRaisedCallback> = self
.subscribers
.read()
.iter()
.map(|(_, callback)| callback.clone())
.collect();
for callback in subscribers {
callback(&[]);
}
}
fn mark_poisoned(&self) -> bool {
let _gate = self.lock_poison_gate();
mark_poisoned(&self.backing_id, &self.path)
}
fn clear_poison(&self) {
let _gate = self.lock_poison_gate();
clear_poison(&self.backing_id, &self.path);
}
fn lock_poison_gate(&self) -> MutexGuard<'_, ()> {
self.run_poison_blocking_hook();
match self.poison_gate.try_lock() {
Some(guard) => guard,
None => {
#[cfg(test)]
{
let hook = self.poison_contended_hook.lock().clone();
if let Some(hook) = hook {
hook();
}
}
self.poison_gate.lock()
}
}
}
fn run_poison_blocking_hook(&self) {
#[cfg(test)]
{
let hook = self.poison_blocking_hook.lock().clone();
if let Some(hook) = hook {
hook();
}
}
}
fn remove_subscriber(&self, token: u64) {
self.subscribers.write().retain(|(t, _)| *t != token);
}
}
fn with_live_poison_gate<R>(id: &BackingId, mutate: impl FnOnce() -> R) -> R {
let existing = {
let guard = core_registry().lock();
guard.cores.get(id).and_then(std::sync::Weak::upgrade)
};
match existing {
Some(core) => {
let _gate = core.lock_poison_gate();
mutate()
}
None => mutate(),
}
}
struct SubscriptionLease {
state: Mutex<LeaseState>,
drained: Condvar,
}
struct LeaseState {
dead: bool,
in_flight: usize,
}
thread_local! {
static ACTIVE_LEASES: std::cell::RefCell<Vec<*const SubscriptionLease>> =
const { std::cell::RefCell::new(Vec::new()) };
}
impl SubscriptionLease {
fn new() -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(LeaseState {
dead: false,
in_flight: 0,
}),
drained: Condvar::new(),
})
}
fn enter(self: &Arc<Self>) -> bool {
{
let mut st = self.state.lock();
if st.dead {
return false;
}
st.in_flight += 1;
}
let ptr = Arc::as_ptr(self);
ACTIVE_LEASES.with(|a| a.borrow_mut().push(ptr));
true
}
fn leave(self: &Arc<Self>) {
let ptr = Arc::as_ptr(self);
ACTIVE_LEASES.with(|a| {
let mut v = a.borrow_mut();
if let Some(i) = v.iter().rposition(|&p| p == ptr) {
v.remove(i);
}
});
let mut st = self.state.lock();
st.in_flight -= 1;
if st.in_flight == 0 {
self.drained.notify_all();
}
}
fn kill_and_drain(self: &Arc<Self>) {
let ptr = Arc::as_ptr(self);
let own_frames = ACTIVE_LEASES.with(|a| a.borrow().iter().filter(|&&p| p == ptr).count());
let mut st = self.state.lock();
st.dead = true;
if own_frames > 0 {
return;
}
while st.in_flight > 0 {
self.drained.wait(&mut st);
}
}
}
#[must_use = "dropping the RaiseSubscription immediately unsubscribes and drains the callback"]
pub struct RaiseSubscription {
core: Weak<StoreCore>,
token: u64,
lease: Arc<SubscriptionLease>,
}
impl Drop for RaiseSubscription {
fn drop(&mut self) {
self.lease.kill_and_drain();
if let Some(core) = self.core.upgrade() {
core.remove_subscriber(self.token);
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
enum BackingId {
FileId { device: u64, inode: u64 },
#[cfg_attr(windows, allow(dead_code))]
Path(PathBuf),
}
impl BackingId {
fn of(lock: &std::fs::File, path: &Path) -> Result<Self, OrgRevocationError> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
Ok(match lock.metadata() {
Ok(meta) => BackingId::FileId {
device: meta.dev(),
inode: meta.ino(),
},
Err(_) => BackingId::Path(path.to_path_buf()),
})
}
#[cfg(windows)]
{
match windows_file_identity(lock) {
Ok((device, inode, _links)) => Ok(BackingId::FileId { device, inode }),
Err(e) => Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason: format!("state lock: cannot read Windows file identity: {e}"),
}),
}
}
#[cfg(not(any(unix, windows)))]
{
let _ = lock;
Ok(BackingId::Path(path.to_path_buf()))
}
}
}
#[cfg(windows)]
fn windows_file_identity(file: &std::fs::File) -> std::io::Result<(u64, u64, u32)> {
use std::os::windows::io::AsRawHandle;
#[repr(C)]
#[derive(Default)]
struct ByHandleFileInformation {
dw_file_attributes: u32,
ft_creation_time: [u32; 2],
ft_last_access_time: [u32; 2],
ft_last_write_time: [u32; 2],
dw_volume_serial_number: u32,
n_file_size_high: u32,
n_file_size_low: u32,
n_number_of_links: u32,
n_file_index_high: u32,
n_file_index_low: u32,
}
extern "system" {
fn GetFileInformationByHandle(
h_file: *mut std::ffi::c_void,
lp_file_information: *mut ByHandleFileInformation,
) -> i32;
}
let mut info = ByHandleFileInformation::default();
let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) };
if ok == 0 {
return Err(std::io::Error::last_os_error());
}
let volume = u64::from(info.dw_volume_serial_number);
let index = (u64::from(info.n_file_index_high) << 32) | u64::from(info.n_file_index_low);
Ok((volume, index, info.n_number_of_links))
}
struct CoreRegistry {
cores: std::collections::HashMap<BackingId, std::sync::Weak<StoreCore>>,
bindings: std::collections::HashMap<PathBuf, BackingId>,
}
static CORES: std::sync::OnceLock<Mutex<CoreRegistry>> = std::sync::OnceLock::new();
fn core_registry() -> &'static Mutex<CoreRegistry> {
CORES.get_or_init(|| {
Mutex::new(CoreRegistry {
cores: std::collections::HashMap::new(),
bindings: std::collections::HashMap::new(),
})
})
}
fn join_or_create_core(
backing_id: BackingId,
path: &Path,
disk: OrgRevocationState,
) -> Result<(Arc<StoreCore>, Vec<RaisedFloor>), OrgRevocationError> {
let mut guard = core_registry().lock();
let reg = &mut *guard;
reg.cores.retain(|_, weak| weak.strong_count() > 0);
let live_ids = ®.cores;
reg.bindings.retain(|_, id| live_ids.contains_key(id));
if let Some(bound) = reg.bindings.get(path) {
if *bound != backing_id {
return Err(OrgRevocationError::BackingIdentityConflict {
path: path.display().to_string(),
});
}
}
reg.bindings.insert(path.to_path_buf(), backing_id.clone());
let existing = reg
.cores
.get(&backing_id)
.and_then(std::sync::Weak::upgrade);
if let Some(core) = existing {
drop(guard);
let raised = {
let _reload = core.reload.lock();
core.publish(disk)
};
return Ok((core, raised));
}
let core = Arc::new(StoreCore {
path: path.to_path_buf(),
backing_id: backing_id.clone(),
reload: Mutex::new(()),
live: RwLock::new(Arc::new(disk)),
generation: AtomicU64::new(0),
generation_exhausted: AtomicBool::new(false),
poison_gate: Mutex::new(()),
#[cfg(test)]
publish_contended_hook: Mutex::new(None),
#[cfg(test)]
poison_blocking_hook: Mutex::new(None),
#[cfg(test)]
force_post_rename: AtomicBool::new(false),
#[cfg(test)]
poison_contended_hook: Mutex::new(None),
subscribers: RwLock::new(Vec::new()),
next_subscriber: AtomicU64::new(0),
#[cfg(any(test, feature = "fixtures"))]
publish_pause: parking_lot::Mutex::new(None),
});
reg.cores.insert(backing_id, Arc::downgrade(&core));
Ok((core, Vec::new()))
}
pub(crate) struct PublishGuard<'a> {
_guards: Vec<parking_lot::MutexGuard<'a, ()>>,
}
pub(crate) fn publish_guard_pair<'a>(
a: &'a OrgRevocationStore,
b: &'a OrgRevocationStore,
) -> PublishGuard<'a> {
if Arc::ptr_eq(&a.core, &b.core) {
return PublishGuard {
_guards: vec![a.core.reload.lock()],
};
}
let (first, second) = if a.core.path <= b.core.path {
(a, b)
} else {
(b, a)
};
let g1 = first.core.reload.lock();
let g2 = second.core.reload.lock();
PublishGuard {
_guards: vec![g1, g2],
}
}
pub struct PublicationPin<'a> {
core: &'a StoreCore,
_poison: parking_lot::MutexGuard<'a, ()>,
_live: parking_lot::RwLockReadGuard<'a, Arc<OrgRevocationState>>,
}
impl PublicationPin<'_> {
pub fn generation(&self) -> Result<BarrieredGeneration, GenerationExhausted> {
OrgRevocationStore::sample_generation(self.core)
}
pub fn poisoned(&self) -> bool {
is_poisoned(&self.core.backing_id, &self.core.path)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct BarrieredGeneration(u64);
impl BarrieredGeneration {
pub fn get(self) -> u64 {
self.0
}
#[doc(hidden)]
#[cfg(any(test, feature = "fixtures"))]
pub fn from_raw_for_test(raw: u64) -> Self {
Self(raw)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GenerationExhausted;
impl std::fmt::Display for GenerationExhausted {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("org revocation publication generation space is exhausted")
}
}
impl std::error::Error for GenerationExhausted {}
pub struct OrgRevocationStore {
core: Arc<StoreCore>,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum ProvisioningExpectation {
MayBeFresh,
MustExist,
}
impl OrgRevocationStore {
pub fn init(
path: impl Into<PathBuf>,
expect: ProvisioningExpectation,
) -> Result<Self, OrgRevocationError> {
let path = path.into();
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent).map_err(|e| OrgRevocationError::Io {
path: path.display().to_string(),
reason: e.to_string(),
})?;
}
}
let path = normalize_backing_path(&path)?;
let sidecar_predates_us = {
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
std::fs::symlink_metadata(PathBuf::from(lock_path)).is_ok()
};
let lock = lock_state_file(&path)?;
let backing_id = BackingId::of(&lock, &path)?;
let was_poisoned = is_poisoned(&backing_id, &path);
if was_poisoned {
prove_entry_durable(&path)?;
}
let state = match read_regular_nofollow(&path) {
Ok(bytes) => OrgRevocationState::from_file_bytes(&bytes, &path)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
if expect == ProvisioningExpectation::MustExist || sidecar_predates_us {
let err = OrgRevocationError::MissingState {
path: path.display().to_string(),
};
tracing::error!(
sidecar_predates_us,
?expect,
"{err}; refusing to re-create it as EMPTY — that would \
discard every revocation floor and re-admit every \
certificate this org has revoked. Restore the state \
file from backup, or remove the whole authority \
directory to provision deliberately from scratch."
);
return Err(err);
}
let state = OrgRevocationState::empty();
match write_atomic_phased(&path, &state.to_file_bytes()?) {
Ok(()) => {}
Err(WritePhase::PreRename(reason)) => {
return Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason,
})
}
Err(WritePhase::PostRename(reason)) => {
with_live_poison_gate(&backing_id, || {
let _ = mark_poisoned(&backing_id, &path);
});
return Err(OrgRevocationError::DurabilityUncertain {
path: path.display().to_string(),
reason,
});
}
}
state
}
Err(e) => {
return Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason: e.to_string(),
})
}
};
let (core, raised) = join_or_create_core(backing_id.clone(), &path, state)?;
if was_poisoned {
core.clear_poison();
}
drop(lock);
let store = Self { core };
store.core.notify(&raised);
if was_poisoned {
store.core.notify_authority_changed();
}
Ok(store)
}
pub fn open_existing(path: impl Into<PathBuf>) -> Result<Self, OrgRevocationError> {
let path = normalize_backing_path(&path.into())?;
let lock = lock_state_file(&path)?;
let backing_id = BackingId::of(&lock, &path)?;
let was_poisoned = is_poisoned(&backing_id, &path);
if was_poisoned {
prove_entry_durable(&path)?;
}
let bytes = match read_regular_nofollow(&path) {
Ok(bytes) => bytes,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
let err = OrgRevocationError::MissingState {
path: path.display().to_string(),
};
tracing::error!("{err}");
return Err(err);
}
Err(e) => {
return Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason: e.to_string(),
})
}
};
let state = OrgRevocationState::from_file_bytes(&bytes, &path).inspect_err(|err| {
tracing::error!("{err}");
})?;
let (core, raised) = join_or_create_core(backing_id.clone(), &path, state)?;
if was_poisoned {
core.clear_poison();
}
drop(lock);
let store = Self { core };
store.core.notify(&raised);
if was_poisoned {
store.core.notify_authority_changed();
}
Ok(store)
}
pub fn path(&self) -> &Path {
&self.core.path
}
pub fn snapshot(&self) -> Arc<OrgRevocationState> {
self.core.live.read().clone()
}
pub fn floor_for(&self, org: &OrgId, member: &EntityId) -> u32 {
self.snapshot().floor_for(org, member)
}
pub fn is_poisoned(&self) -> bool {
is_poisoned(&self.core.backing_id, &self.core.path)
}
pub fn generation_exhausted_for_metrics(&self) -> bool {
self.core.generation_exhausted.load(Ordering::Acquire)
}
#[doc(hidden)]
#[cfg(any(test, feature = "fixtures"))]
pub fn saturate_generation_for_test(&self) {
self.core.generation.store(u64::MAX, Ordering::Release);
}
#[doc(hidden)]
#[cfg(test)]
pub(crate) fn arm_publish_contended_hook(&self, hook: Arc<dyn Fn() + Send + Sync>) {
*self.core.publish_contended_hook.lock() = Some(hook);
}
#[doc(hidden)]
#[cfg(any(test, feature = "fixtures"))]
pub fn republish_for_test(&self) {
let current = (*self.core.live.read()).as_ref().clone();
self.core.publish(current);
}
#[doc(hidden)]
#[cfg(any(test, feature = "fixtures"))]
pub fn mark_poisoned_for_test(&self) {
self.core.mark_poisoned();
}
#[doc(hidden)]
#[cfg(test)]
pub(crate) fn arm_poison_blocking_hook(&self, hook: Arc<dyn Fn() + Send + Sync>) {
*self.core.poison_blocking_hook.lock() = Some(hook);
}
#[doc(hidden)]
#[cfg(test)]
pub(crate) fn arm_poison_contended_hook(&self, hook: Arc<dyn Fn() + Send + Sync>) {
*self.core.poison_contended_hook.lock() = Some(hook);
}
#[doc(hidden)]
#[cfg(test)]
pub(crate) fn arm_forced_post_rename_for_test(&self) {
self.core.force_post_rename.store(true, Ordering::Release);
}
pub fn shares_core_with(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.core, &other.core)
}
#[allow(dead_code)]
pub(crate) fn publish_generation(&self) -> u64 {
self.core.generation.load(Ordering::Acquire)
}
pub fn barriered_generation(&self) -> Result<BarrieredGeneration, GenerationExhausted> {
let _live = self.core.live.read();
Self::sample_generation(&self.core)
}
pub fn pin_publication(&self) -> PublicationPin<'_> {
let poison = self.core.poison_gate.lock();
let live = self.core.live.read();
PublicationPin {
core: &self.core,
_poison: poison,
_live: live,
}
}
fn sample_generation(core: &StoreCore) -> Result<BarrieredGeneration, GenerationExhausted> {
if core.generation_exhausted.load(Ordering::Acquire) {
return Err(GenerationExhausted);
}
Ok(BarrieredGeneration(core.generation.load(Ordering::Acquire)))
}
pub fn snapshot_with_generation(
&self,
) -> Result<(Arc<OrgRevocationState>, BarrieredGeneration), GenerationExhausted> {
let live = self.core.live.read();
let generation = Self::sample_generation(&self.core)?;
Ok((live.clone(), generation))
}
#[doc(hidden)]
#[cfg(any(test, feature = "fixtures"))]
pub fn arm_publish_pause_for_test(
&self,
) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
let (swapped_tx, swapped_rx) = std::sync::mpsc::channel();
let (resume_tx, resume_rx) = std::sync::mpsc::channel();
*self.core.publish_pause.lock() = Some(PublishPauseHook {
swapped: swapped_tx,
resume: resume_rx,
});
(swapped_rx, resume_tx)
}
pub(crate) fn publish_guard(&self) -> PublishGuard<'_> {
PublishGuard {
_guards: vec![self.core.reload.lock()],
}
}
#[doc(hidden)]
pub fn subscriber_count(&self) -> usize {
self.core.subscribers.read().len()
}
#[doc(hidden)]
pub fn snapshot_subscribers_for_test(&self) -> Vec<FloorsRaisedCallback> {
self.core
.subscribers
.read()
.iter()
.map(|(_, callback)| callback.clone())
.collect()
}
#[must_use = "dropping the returned guard immediately unsubscribes the callback"]
pub fn subscribe_floors_raised(
&self,
callback: impl Fn(&[RaisedFloor]) + Send + Sync + 'static,
) -> RaiseSubscription {
let lease = SubscriptionLease::new();
let lease_cb = Arc::clone(&lease);
let wrapped: FloorsRaisedCallback = Arc::new(move |raised: &[RaisedFloor]| {
if !lease_cb.enter() {
return;
}
struct LeaveOnDrop<'a>(&'a Arc<SubscriptionLease>);
impl Drop for LeaveOnDrop<'_> {
fn drop(&mut self) {
self.0.leave();
}
}
let _leave = LeaveOnDrop(&lease_cb);
callback(raised);
});
let token = self.core.next_subscriber.fetch_add(1, Ordering::Relaxed);
self.core.subscribers.write().push((token, wrapped));
RaiseSubscription {
core: Arc::downgrade(&self.core),
token,
lease,
}
}
pub fn apply_bundle(
&self,
bundle: &OrgRevocationBundle,
) -> Result<Vec<RaisedFloor>, OrgRevocationError> {
let path = &self.core.path;
enum LockedOutcome {
Applied(Vec<RaisedFloor>, bool),
DurabilityUncertain(Vec<RaisedFloor>, String, bool),
}
if let Err(e) = bundle.verify() {
let err = OrgRevocationError::InvalidBundle(e);
tracing::error!(
org = %bundle.org_id,
"rejecting revocation bundle, keeping last-good persisted floors: {err}"
);
return Err(err);
}
let outcome = {
let lock = lock_state_file(path)?;
let _guard = self.core.reload.lock();
let opened_id = BackingId::of(&lock, path)?;
if opened_id != self.core.backing_id {
drop(lock);
return Err(OrgRevocationError::BackingIdentityConflict {
path: path.display().to_string(),
});
}
let was_poisoned = is_poisoned(&self.core.backing_id, path);
if was_poisoned {
prove_entry_durable(path)?;
}
let disk_bytes = read_regular_nofollow(path).map_err(|e| OrgRevocationError::Io {
path: path.display().to_string(),
reason: e.to_string(),
})?;
let disk = OrgRevocationState::from_file_bytes(&disk_bytes, path)?;
let mut merged = disk.clone();
let raised_on_disk = merged.merge_bundle(bundle);
let mut durability_uncertain: Option<(String, bool)> = None;
if raised_on_disk > 0 {
#[cfg(test)]
let write = if self.core.force_post_rename.swap(false, Ordering::AcqRel) {
Err(WritePhase::PostRename(
"forced post-rename failure (test seam)".to_string(),
))
} else {
write_atomic_phased(path, &merged.to_file_bytes()?)
};
#[cfg(not(test))]
let write = write_atomic_phased(path, &merged.to_file_bytes()?);
match write {
Ok(()) => {}
Err(WritePhase::PreRename(reason)) => {
drop(lock);
return Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason,
});
}
Err(WritePhase::PostRename(reason)) => {
let newly_poisoned = self.core.mark_poisoned();
durability_uncertain = Some((reason, newly_poisoned));
}
}
}
let raised = self.core.publish(merged);
let recovered = was_poisoned && durability_uncertain.is_none();
if recovered {
self.core.clear_poison();
}
drop(lock);
match durability_uncertain {
None => LockedOutcome::Applied(raised, recovered),
Some((reason, newly)) => LockedOutcome::DurabilityUncertain(raised, reason, newly),
}
};
match outcome {
LockedOutcome::Applied(raised, recovered) => {
self.core.notify(&raised);
if recovered {
self.core.notify_authority_changed();
}
Ok(raised)
}
LockedOutcome::DurabilityUncertain(raised, reason, newly_poisoned) => {
let err = OrgRevocationError::DurabilityUncertain {
path: path.display().to_string(),
reason,
};
tracing::error!("{err}");
self.core.notify(&raised);
if newly_poisoned && raised.is_empty() {
self.core.notify_authority_changed();
}
Err(err)
}
}
}
}
impl std::fmt::Debug for OrgRevocationStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OrgRevocationStore")
.field("path", &self.core.path)
.field("floors", &self.snapshot().len())
.finish()
}
}
pub(crate) enum WritePhase {
PreRename(String),
PostRename(String),
}
static PATH_POISON: std::sync::OnceLock<Mutex<PoisonRegistry>> = std::sync::OnceLock::new();
#[derive(Default)]
struct PoisonRegistry {
by_id: std::collections::HashSet<BackingId>,
by_path: std::collections::HashMap<PathBuf, std::collections::HashSet<BackingId>>,
}
fn poison_registry() -> &'static Mutex<PoisonRegistry> {
PATH_POISON.get_or_init(|| Mutex::new(PoisonRegistry::default()))
}
static POISON_KEY_MEMO: std::sync::OnceLock<Mutex<std::collections::HashMap<PathBuf, PathBuf>>> =
std::sync::OnceLock::new();
fn poison_key_memo() -> &'static Mutex<std::collections::HashMap<PathBuf, PathBuf>> {
POISON_KEY_MEMO.get_or_init(|| Mutex::new(std::collections::HashMap::new()))
}
fn poison_path_key(normalized_path: &Path) -> PathBuf {
if let Some(hit) = poison_key_memo().lock().get(normalized_path) {
return hit.clone();
}
match std::fs::canonicalize(normalized_path) {
Ok(canonical) => {
poison_key_memo()
.lock()
.insert(normalized_path.to_path_buf(), canonical.clone());
canonical
}
Err(_) => normalized_path.to_path_buf(),
}
}
fn mark_poisoned(id: &BackingId, normalized_path: &Path) -> bool {
let key = poison_path_key(normalized_path);
let mut reg = poison_registry().lock();
let newly = !reg.by_id.contains(id) && !reg.by_path.contains_key(&key);
reg.by_id.insert(id.clone());
reg.by_path.entry(key).or_default().insert(id.clone());
newly
}
fn is_poisoned(id: &BackingId, normalized_path: &Path) -> bool {
let key = poison_path_key(normalized_path);
let reg = poison_registry().lock();
reg.by_id.contains(id) || reg.by_path.contains_key(&key)
}
pub(crate) fn normalize_backing_path(path: &Path) -> Result<PathBuf, OrgRevocationError> {
let io = |reason: String| OrgRevocationError::Io {
path: path.display().to_string(),
reason,
};
let Some(file_name) = path.file_name() else {
return Err(io("backing path has no final component".to_string()));
};
let parent = match path.parent() {
Some(p) if !p.as_os_str().is_empty() => p,
_ => Path::new("."),
};
let canon_parent = parent
.canonicalize()
.map_err(|e| io(format!("cannot canonicalize parent directory: {e}")))?;
let joined = canon_parent.join(file_name);
match open_regular_nofollow(&joined) {
Ok(_) => Ok(joined),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(joined),
Err(e) => Err(io(format!(
"refusing non-regular backing path (symlink/FIFO/other): {e}"
))),
}
}
fn prove_entry_durable(path: &Path) -> Result<(), OrgRevocationError> {
fsync_parent_dir(path).map_err(|e| {
tracing::error!(
path = %path.display(),
error = %e,
"revocation-state durability recovery failed; path remains poisoned"
);
OrgRevocationError::Poisoned {
path: path.display().to_string(),
}
})
}
fn clear_poison(id: &BackingId, path: &Path) {
let key = poison_path_key(path);
{
let mut reg = poison_registry().lock();
if let Some(ids) = reg.by_path.remove(&key) {
for stale in ids {
reg.by_id.remove(&stale);
}
}
reg.by_id.remove(id);
}
tracing::warn!(
path = %path.display(),
"revocation-state durability uncertainty recovered \
(locked reread republished; parent directory fsynced)"
);
}
pub(crate) fn open_regular_nofollow(path: &Path) -> std::io::Result<std::fs::File> {
let mut opts = std::fs::OpenOptions::new();
opts.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
}
#[cfg(not(unix))]
{
let meta = std::fs::symlink_metadata(path)?;
if meta.file_type().is_symlink() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"refusing symlink: authority files must be regular files",
));
}
}
let opened = opts.open(path);
#[cfg(unix)]
let opened = opened.map_err(|e| {
if e.raw_os_error() == Some(libc::ELOOP) {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"refusing symlink: authority files must be regular files",
)
} else {
e
}
});
let file = opened?;
let meta = file.metadata()?;
if !meta.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"refusing non-regular file: authority files must be regular files",
));
}
Ok(file)
}
pub(crate) fn read_regular_nofollow(path: &Path) -> std::io::Result<Vec<u8>> {
use std::io::Read;
let mut file = open_regular_nofollow(path)?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)?;
Ok(bytes)
}
pub(crate) fn lock_state_file(path: &Path) -> Result<std::fs::File, OrgRevocationError> {
let io = |e: std::io::Error| OrgRevocationError::Io {
path: path.display().to_string(),
reason: format!("state lock: {e}"),
};
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock = open_lock_file(&PathBuf::from(lock_path)).map_err(io)?;
#[cfg(unix)]
let nlink = {
use std::os::unix::fs::MetadataExt;
lock.metadata().map_err(io)?.nlink()
};
#[cfg(windows)]
let nlink = {
let (_volume, _index, links) = windows_file_identity(&lock).map_err(io)?;
u64::from(links)
};
#[cfg(any(unix, windows))]
if nlink != 1 {
return Err(OrgRevocationError::Io {
path: path.display().to_string(),
reason: format!(
"state lock: refusing .lock sidecar with {nlink} hard links \
(expected 1) — a hard-linked sidecar would alias two backing paths"
),
});
}
Ok(lock)
}
pub(crate) fn open_lock_file(lock_path: &Path) -> std::io::Result<std::fs::File> {
let mut opts = std::fs::OpenOptions::new();
opts.create(true).write(true).truncate(false);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK);
opts.mode(0o600);
}
#[cfg(not(unix))]
{
if let Ok(meta) = std::fs::symlink_metadata(lock_path) {
if meta.file_type().is_symlink() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"refusing symlink: lock files must be regular files",
));
}
}
}
let f = opts.open(lock_path)?;
if !f.metadata()?.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"refusing non-regular file: lock files must be regular files",
));
}
f.lock()?;
Ok(f)
}
fn fsync_parent_dir(path: &Path) -> std::io::Result<()> {
#[cfg(unix)]
{
let dir = match path.parent() {
Some(p) if !p.as_os_str().is_empty() => p,
_ => Path::new("."),
};
std::fs::File::open(dir)?.sync_all()?;
}
#[cfg(not(unix))]
{
let _ = path;
}
Ok(())
}
#[cfg(windows)]
#[allow(clippy::multiple_unsafe_ops_per_block)]
fn rename_write_through(src: &Path, dest: &Path) -> std::io::Result<()> {
use std::os::windows::ffi::OsStrExt;
extern "system" {
fn MoveFileExW(existing: *const u16, new: *const u16, flags: u32) -> i32;
}
const MOVEFILE_REPLACE_EXISTING: u32 = 0x0000_0001;
const MOVEFILE_WRITE_THROUGH: u32 = 0x0000_0008;
let mut from: Vec<u16> = src.as_os_str().encode_wide().collect();
from.push(0);
let mut to: Vec<u16> = dest.as_os_str().encode_wide().collect();
to.push(0);
let ok = unsafe {
MoveFileExW(
from.as_ptr(),
to.as_ptr(),
MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
)
};
if ok == 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
}
static TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
fn fresh_temp_path(path: &Path) -> Result<PathBuf, WritePhase> {
let mut rand = [0u8; 8];
getrandom::fill(&mut rand)
.map_err(|e| WritePhase::PreRename(format!("temp-name entropy unavailable: {e:?}")))?;
let mut s = path.as_os_str().to_os_string();
s.push(format!(
".tmp.{}.{}.{}",
std::process::id(),
TEMP_SEQ.fetch_add(1, Ordering::Relaxed),
hex::encode(rand)
));
Ok(PathBuf::from(s))
}
pub(crate) fn write_atomic_phased(path: &Path, bytes: &[u8]) -> Result<(), WritePhase> {
let pre = |e: std::io::Error| WritePhase::PreRename(e.to_string());
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent).map_err(pre)?;
}
}
let mut tmp = fresh_temp_path(path)?;
let mut file = None;
for _ in 0..4 {
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
match opts.open(&tmp) {
Ok(f) => {
file = Some(f);
break;
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
tmp = fresh_temp_path(path)?;
}
Err(e) => return Err(pre(e)),
}
}
let Some(mut f) = file else {
return Err(WritePhase::PreRename(
"could not create a fresh temp file after 4 attempts".to_string(),
));
};
let write_result = (|| -> std::io::Result<()> {
use std::io::Write;
f.write_all(bytes)?;
f.flush()?;
f.sync_all()?;
Ok(())
})();
if let Err(e) = write_result {
drop(f);
let _ = std::fs::remove_file(&tmp);
return Err(pre(e));
}
drop(f);
#[cfg(windows)]
let renamed = rename_write_through(&tmp, path);
#[cfg(not(windows))]
let renamed = std::fs::rename(&tmp, path);
if let Err(e) = renamed {
let _ = std::fs::remove_file(&tmp);
return Err(pre(e));
}
if let Err(e) = fsync_parent_dir(path) {
return Err(WritePhase::PostRename(e.to_string()));
}
Ok(())
}
pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), OrgRevocationError> {
write_atomic_phased(path, bytes).map_err(|phase| match phase {
WritePhase::PreRename(reason) => OrgRevocationError::Io {
path: path.display().to_string(),
reason,
},
WritePhase::PostRename(reason) => OrgRevocationError::DurabilityUncertain {
path: path.display().to_string(),
reason,
},
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::adapter::net::behavior::org::OrgKeypair;
use std::sync::atomic::{AtomicUsize, Ordering};
static TEST_DIR_SEQ: AtomicUsize = AtomicUsize::new(0);
#[test]
fn parsed_state_drops_zero_floor_rows() {
let scratch = Scratch::new();
let path = scratch.state_path();
let org_id = org().org_id();
let live = member();
let null = EntityId::from_bytes([0xEE; 32]);
let json = format!(
r#"{{"version":{ORG_REVOCATION_STATE_VERSION},"floors":[
{{"org":"{org_hex}","member":"{live_hex}","floor":7}},
{{"org":"{org_hex}","member":"{null_hex}","floor":0}}
]}}"#,
org_hex = hex::encode(org_id.as_bytes()),
live_hex = hex::encode(live.as_bytes()),
null_hex = hex::encode(null.as_bytes()),
);
std::fs::write(&path, json).expect("write hand-made state");
let state =
OrgRevocationState::from_file_bytes(&std::fs::read(&path).expect("read back"), &path)
.expect("a zero row must not make the file unloadable");
assert_eq!(
state.floor_for(&org_id, &live),
7,
"the real floor survives"
);
assert_eq!(
state.floor_for(&org_id, &null),
0,
"a zero floor reads as the implicit default either way",
);
assert_eq!(
state.iter().count(),
1,
"the zero row must not be MATERIALIZED — it is what accumulates \
and makes every authority install take an exclusive fold lock \
per entry",
);
}
#[test]
fn a_zero_floor_is_not_persisted() {
let org = org();
let zero_member = crate::adapter::net::identity::EntityKeypair::generate()
.entity_id()
.clone();
let real_member = crate::adapter::net::identity::EntityKeypair::generate()
.entity_id()
.clone();
let mut map = BTreeMap::new();
map.insert(zero_member.clone(), 0u32);
map.insert(real_member.clone(), 3u32);
let bundle = OrgRevocationBundle::try_issue(&org, &map).expect("bundle");
let mut state = OrgRevocationState::empty();
let raised = state.merge_bundle(&bundle);
assert_eq!(raised, 1, "only the nonzero floor counts as a raise");
assert_eq!(
state.floors.len(),
1,
"the zero floor must not materialize a row; got {:?}",
state.floors,
);
assert_eq!(state.floor_for(&org.org_id(), &zero_member), 0);
assert_eq!(state.floor_for(&org.org_id(), &real_member), 3);
let mut map = BTreeMap::new();
map.insert(zero_member.clone(), 5u32);
let bundle = OrgRevocationBundle::try_issue(&org, &map).expect("bundle");
assert_eq!(state.merge_bundle(&bundle), 1);
assert_eq!(state.floor_for(&org.org_id(), &zero_member), 5);
}
#[test]
fn poison_key_memo_does_not_cache_the_pre_creation_fallback() {
let scratch = Scratch::new();
let indirect = scratch.0.join("sub").join("..").join("state.json");
std::fs::create_dir_all(scratch.0.join("sub")).expect("mkdir");
let before = poison_path_key(&indirect);
assert_eq!(
before,
indirect.to_path_buf(),
"a missing file falls back to the normalized path",
);
std::fs::write(&indirect, b"{}").expect("write state");
let after = poison_path_key(&indirect);
let expected = std::fs::canonicalize(&indirect).expect("canonicalize");
assert_eq!(
after, expected,
"once the file exists the canonical key must be used",
);
assert_ne!(
after, before,
"the pre-creation fallback must not have been memoized",
);
assert_eq!(poison_path_key(&indirect), expected, "memo is stable");
}
struct Scratch(PathBuf);
impl Scratch {
fn new() -> Self {
let dir = std::env::temp_dir().join(format!(
"net-org-revocation-{}-{}",
std::process::id(),
TEST_DIR_SEQ.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&dir).expect("create scratch dir");
Self(dir)
}
fn state_path(&self) -> PathBuf {
self.0.join("revocation-state.json")
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn org() -> OrgKeypair {
OrgKeypair::from_bytes([0x42u8; 32])
}
fn member() -> EntityId {
EntityId::from_bytes([0x24u8; 32])
}
fn bundle_with_floor(generation: u32) -> OrgRevocationBundle {
let mut floors = BTreeMap::new();
floors.insert(member(), generation);
OrgRevocationBundle::try_issue(&org(), &floors).expect("issue")
}
#[test]
fn case_aliased_paths_share_one_core_on_case_insensitive_fs() {
let scratch = Scratch::new();
let lower = scratch.0.join("revocation-state.json");
let upper = scratch.0.join("REVOCATION-STATE.JSON");
let a = OrgRevocationStore::init(&lower, ProvisioningExpectation::MayBeFresh)
.expect("init lower alias");
if !upper.exists() {
return;
}
let b = OrgRevocationStore::open_existing(&upper).expect("open upper alias");
assert!(
a.shares_core_with(&b),
"case-aliases on a case-insensitive FS must share ONE core (same .lock inode)",
);
a.apply_bundle(&bundle_with_floor(5))
.expect("apply floor via lower alias");
assert_eq!(
b.floor_for(&org().org_id(), &member()),
5,
"a floor published through one alias must be visible through the other",
);
a.mark_poisoned_for_test();
assert!(
b.is_poisoned(),
"poison under one alias must be visible through the other",
);
}
#[test]
fn init_creates_empty_state_and_open_existing_loads_it() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
assert!(store.snapshot().is_empty());
assert!(path.exists());
let reopened = OrgRevocationStore::open_existing(&path).expect("open");
assert!(reopened.snapshot().is_empty());
}
#[test]
fn open_existing_refuses_missing_state() {
let scratch = Scratch::new();
let err = OrgRevocationStore::open_existing(scratch.state_path())
.expect_err("missing file must be loud");
assert!(matches!(err, OrgRevocationError::MissingState { .. }));
}
#[test]
fn apply_bundle_raises_persists_and_publishes() {
let scratch = Scratch::new();
let store =
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init");
let raised = store.apply_bundle(&bundle_with_floor(5)).expect("apply");
assert_eq!(raised, vec![(org().org_id(), member(), 5)]);
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
drop(store);
let reopened = OrgRevocationStore::open_existing(scratch.state_path()).expect("open");
assert_eq!(reopened.floor_for(&org().org_id(), &member()), 5);
}
#[test]
fn restart_witness_lower_valid_bundle_never_rolls_back() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
drop(store);
let store = OrgRevocationStore::open_existing(&path).expect("restart");
let before = std::fs::read(&path).expect("read state");
let raised = store
.apply_bundle(&bundle_with_floor(3))
.expect("valid lower bundle is not an error");
assert!(raised.is_empty(), "lower floor must not merge");
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
assert_eq!(std::fs::read(&path).expect("read state"), before);
drop(store);
let store = OrgRevocationStore::open_existing(&path).expect("restart 2");
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
}
#[test]
fn corrupt_incoming_bundle_keeps_last_good() {
let scratch = Scratch::new();
let store =
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
let mut evil = bundle_with_floor(9);
evil.signature[0] ^= 1;
let before = std::fs::read(store.path()).expect("read state");
let err = store
.apply_bundle(&evil)
.expect_err("tampered bundle rejected");
assert!(matches!(err, OrgRevocationError::InvalidBundle(_)));
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
assert_eq!(std::fs::read(store.path()).expect("read state"), before);
}
#[test]
fn corrupt_persisted_state_is_loud_at_startup() {
let scratch = Scratch::new();
let path = scratch.state_path();
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
std::fs::write(&path, b"{ not json").expect("corrupt");
let err = OrgRevocationStore::open_existing(&path).expect_err("corrupt is loud");
assert!(matches!(err, OrgRevocationError::CorruptState { .. }));
std::fs::write(&path, br#"{"version":99,"floors":[]}"#).expect("write");
let err = OrgRevocationStore::open_existing(&path).expect_err("version is loud");
assert!(matches!(
err,
OrgRevocationError::UnsupportedVersion { found: 99, .. }
));
let org_hex = hex::encode(org().org_id().as_bytes());
let member_hex = hex::encode(member().as_bytes());
let dup = format!(
r#"{{"version":1,"floors":[
{{"org":"{org_hex}","member":"{member_hex}","floor":1}},
{{"org":"{org_hex}","member":"{member_hex}","floor":2}}
]}}"#
);
std::fs::write(&path, dup).expect("write");
let err = OrgRevocationStore::open_existing(&path).expect_err("dup is loud");
assert!(matches!(err, OrgRevocationError::CorruptState { .. }));
}
#[test]
fn init_preserves_existing_maxima_on_readopt() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply");
drop(store);
let readopted =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("re-init");
assert_eq!(readopted.floor_for(&org().org_id(), &member()), 5);
}
#[cfg(windows)]
#[test]
fn write_through_rename_replaces_and_reports_failure() {
let scratch = Scratch::new();
let dest = scratch.state_path();
let src = scratch.0.join("staged.json");
std::fs::write(&dest, b"old").expect("seed destination");
std::fs::write(&src, b"new").expect("seed source");
rename_write_through(&src, &dest).expect("write-through rename must succeed");
assert_eq!(
std::fs::read(&dest).expect("read dest"),
b"new",
"the rename must REPLACE an existing destination — without \
MOVEFILE_REPLACE_EXISTING every republish would fail",
);
assert!(!src.exists(), "the source must be consumed by the move");
let ghost = scratch.0.join("does-not-exist.json");
assert!(
rename_write_through(&ghost, &dest).is_err(),
"a failed move must surface as an error; swallowing it would make \
a lost publish look durable",
);
}
#[test]
fn init_refuses_to_recreate_a_state_file_that_was_deleted() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store = OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh)
.expect("first adopt");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
drop(store);
std::fs::remove_file(&path).expect("remove state file");
let mut sidecar = path.as_os_str().to_os_string();
sidecar.push(".lock");
assert!(
std::fs::symlink_metadata(PathBuf::from(sidecar)).is_ok(),
"precondition: the sidecar must outlive the state file, otherwise \
this test proves nothing about the signal under test",
);
let err = OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh)
.expect_err("a deleted state file must not be re-created as empty");
assert!(
matches!(err, OrgRevocationError::MissingState { .. }),
"expected MissingState, got {err:?}",
);
assert!(
!path.exists(),
"the refusal wrote an empty state anyway — the floors are gone",
);
}
#[test]
fn init_honours_the_callers_must_exist_expectation() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store = OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh)
.expect("first adopt");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
drop(store);
std::fs::remove_file(&path).expect("remove state file");
let mut sidecar = path.as_os_str().to_os_string();
sidecar.push(".lock");
let _ = std::fs::remove_file(PathBuf::from(sidecar));
let err = OrgRevocationStore::init(&path, ProvisioningExpectation::MustExist)
.expect_err("MustExist must refuse an absent state file");
assert!(
matches!(err, OrgRevocationError::MissingState { .. }),
"expected MissingState, got {err:?}",
);
}
#[test]
fn init_still_creates_a_genuinely_fresh_store() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store = OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh)
.expect("a first adopt on a clean path must succeed");
assert_eq!(store.floor_for(&org().org_id(), &member()), 0);
assert!(path.exists(), "a fresh adopt must durably create the state");
}
#[test]
fn merge_is_per_key_monotone_across_orgs_and_members() {
let org_a = OrgKeypair::from_bytes([1u8; 32]);
let org_b = OrgKeypair::from_bytes([2u8; 32]);
let m1 = EntityId::from_bytes([11u8; 32]);
let m2 = EntityId::from_bytes([22u8; 32]);
let mut state = OrgRevocationState::empty();
let mut floors = BTreeMap::new();
floors.insert(m1.clone(), 5);
floors.insert(m2.clone(), 2);
let a1 = OrgRevocationBundle::try_issue(&org_a, &floors).expect("issue");
assert_eq!(state.merge_bundle(&a1), 2);
let b1 = OrgRevocationBundle::try_issue(&org_b, &floors).expect("issue");
assert_eq!(state.merge_bundle(&b1), 2);
assert_eq!(state.floor_for(&org_a.org_id(), &m1), 5);
assert_eq!(state.floor_for(&org_b.org_id(), &m1), 5);
let mut floors = BTreeMap::new();
floors.insert(m1.clone(), 3);
floors.insert(m2.clone(), 7);
let a2 = OrgRevocationBundle::try_issue(&org_a, &floors).expect("issue");
assert_eq!(state.merge_bundle(&a2), 1);
assert_eq!(state.floor_for(&org_a.org_id(), &m1), 5);
assert_eq!(state.floor_for(&org_a.org_id(), &m2), 7);
assert_eq!(
state.floor_for(&org_a.org_id(), &EntityId::from_bytes([99u8; 32])),
0
);
}
#[cfg(unix)]
#[test]
fn persist_failure_never_publishes_the_live_view() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
std::fs::remove_file(&path).expect("remove");
std::fs::create_dir(&path).expect("dir at path");
std::fs::write(path.join("occupied"), b"x").expect("occupy");
let err = store
.apply_bundle(&bundle_with_floor(9))
.expect_err("rename onto non-empty dir must fail");
assert!(matches!(err, OrgRevocationError::Io { .. }));
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
assert!(!store.is_poisoned());
let leftovers: Vec<_> = std::fs::read_dir(&scratch.0)
.expect("read scratch")
.filter_map(|e| e.ok())
.filter(|e| e.file_name().to_string_lossy().contains(".tmp."))
.collect();
assert!(leftovers.is_empty(), "stale temps: {leftovers:?}");
}
fn bundle_for(member: EntityId, generation: u32) -> OrgRevocationBundle {
let mut floors = BTreeMap::new();
floors.insert(member, generation);
OrgRevocationBundle::try_issue(&org(), &floors).expect("issue")
}
#[test]
fn same_path_handles_share_one_live_view_and_preserve_all_maxima() {
let scratch = Scratch::new();
let path = scratch.state_path();
let member_x = EntityId::from_bytes([0xAAu8; 32]);
let member_y = EntityId::from_bytes([0xBBu8; 32]);
let store_a =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init A");
let store_b = OrgRevocationStore::open_existing(&path).expect("open B");
assert!(
store_a.shares_core_with(&store_b),
"same normalized path must join one core"
);
store_a
.apply_bundle(&bundle_for(member_x.clone(), 5))
.expect("A applies x=5");
assert_eq!(store_b.floor_for(&org().org_id(), &member_x), 5);
let raised = store_b
.apply_bundle(&bundle_for(member_y.clone(), 7))
.expect("B applies y=7");
assert_eq!(raised, vec![(org().org_id(), member_y.clone(), 7)]);
let reopened = OrgRevocationStore::open_existing(&path).expect("reopen");
assert_eq!(reopened.floor_for(&org().org_id(), &member_x), 5);
assert_eq!(reopened.floor_for(&org().org_id(), &member_y), 7);
assert!(reopened.shares_core_with(&store_a));
}
#[test]
fn fresh_open_serializes_behind_the_state_lock_and_recovers_poison() {
let scratch = Scratch::new();
let path = scratch.state_path();
drop(OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init"));
let norm = normalize_backing_path(&path).expect("normalize");
let lock = lock_state_file(&norm).expect("lock");
let opener_path = path.clone();
let (started_tx, started_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let opener = std::thread::spawn(move || {
started_tx.send(()).expect("send started");
let result = OrgRevocationStore::open_existing(&opener_path);
done_tx.send(()).expect("send done");
result
});
started_rx
.recv_timeout(std::time::Duration::from_secs(5))
.expect("opener started");
assert!(
done_rx
.recv_timeout(std::time::Duration::from_millis(300))
.is_err(),
"open_existing must serialize behind the state lock"
);
let mut stronger = OrgRevocationState::empty();
stronger.merge_bundle(&bundle_with_floor(9));
write_atomic(&norm, &stronger.to_file_bytes().expect("bytes")).expect("write");
mark_poisoned(&BackingId::of(&lock, &norm).expect("backing id"), &norm);
drop(lock);
let opened = opener
.join()
.expect("join opener")
.expect("open recovers and succeeds");
assert_eq!(opened.floor_for(&org().org_id(), &member()), 9);
assert!(
!opened.is_poisoned(),
"successful recovery clears the path-wide bit"
);
}
#[test]
fn opener_cannot_publish_through_a_held_publish_guard() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store_a =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
let norm = normalize_backing_path(&path).expect("normalize");
let mut stronger = OrgRevocationState::empty();
stronger.merge_bundle(&bundle_with_floor(10));
{
let _lk = lock_state_file(&norm).expect("lock");
write_atomic(&norm, &stronger.to_file_bytes().expect("bytes")).expect("write");
}
let guard = store_a.publish_guard();
let opener_path = path.clone();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let opener = std::thread::spawn(move || {
let s = OrgRevocationStore::open_existing(&opener_path).expect("open");
done_tx.send(()).expect("done");
s
});
assert!(
done_rx
.recv_timeout(std::time::Duration::from_millis(300))
.is_err(),
"opener published inside a held PublishGuard"
);
assert_eq!(
store_a.floor_for(&org().org_id(), &member()),
0,
"the guarded live view must not move under an opener"
);
drop(guard);
let opened = opener.join().expect("join");
assert_eq!(opened.floor_for(&org().org_id(), &member()), 10);
assert_eq!(store_a.floor_for(&org().org_id(), &member()), 10);
}
#[test]
fn multiple_subscribers_on_one_path_all_observe_raises() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store_a =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init A");
let store_b = OrgRevocationStore::open_existing(&path).expect("open B");
let seen_a: Arc<Mutex<Vec<u32>>> = Arc::new(Mutex::new(Vec::new()));
let seen_b: Arc<Mutex<Vec<u32>>> = Arc::new(Mutex::new(Vec::new()));
let seen_tok: Arc<Mutex<Vec<u32>>> = Arc::new(Mutex::new(Vec::new()));
let sink = seen_a.clone();
let _sub_a = store_a.subscribe_floors_raised(move |raised| {
sink.lock().extend(raised.iter().map(|(_, _, f)| *f));
});
let sink = seen_b.clone();
let _sub_b = store_b.subscribe_floors_raised(move |raised| {
sink.lock().extend(raised.iter().map(|(_, _, f)| *f));
});
let sink = seen_tok.clone();
let subscription = store_a.subscribe_floors_raised(move |raised| {
sink.lock().extend(raised.iter().map(|(_, _, f)| *f));
});
store_a
.apply_bundle(&bundle_with_floor(5))
.expect("apply 5");
assert_eq!(*seen_a.lock(), vec![5]);
assert_eq!(*seen_b.lock(), vec![5]);
assert_eq!(*seen_tok.lock(), vec![5]);
drop(subscription);
store_b
.apply_bundle(&bundle_with_floor(7))
.expect("apply 7");
assert_eq!(*seen_a.lock(), vec![5, 7]);
assert_eq!(*seen_b.lock(), vec![5, 7]);
assert_eq!(*seen_tok.lock(), vec![5], "unsubscribed token is silent");
}
#[test]
fn dropping_the_subscription_guard_unsubscribes_while_the_store_lives() {
let scratch = Scratch::new();
let store =
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init");
assert_eq!(store.subscriber_count(), 0);
let subscription = store.subscribe_floors_raised(|_raised| {});
assert_eq!(
store.subscriber_count(),
1,
"subscribe registers one callback"
);
drop(subscription);
assert_eq!(
store.subscriber_count(),
0,
"dropping the guard unsubscribed while the store handle is still alive",
);
}
#[test]
fn teardown_blocks_until_an_in_flight_callback_leaves() {
use std::sync::mpsc;
use std::time::Duration;
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let (entered_tx, entered_rx) = mpsc::channel::<()>();
let (release_tx, release_rx) = mpsc::channel::<()>();
let entered_tx = Mutex::new(entered_tx);
let release_rx = Mutex::new(release_rx);
let ran = Arc::new(AtomicUsize::new(0));
let ran_cb = Arc::clone(&ran);
let subscription = store.subscribe_floors_raised(move |_raised| {
ran_cb.fetch_add(1, Ordering::SeqCst);
entered_tx.lock().send(()).expect("signal entered");
release_rx.lock().recv().expect("await release");
});
let store_fire = Arc::clone(&store);
let fire = std::thread::spawn(move || {
store_fire
.apply_bundle(&bundle_with_floor(5))
.expect("apply 5");
});
entered_rx
.recv_timeout(Duration::from_secs(2))
.expect("callback entered");
let (teardown_done_tx, teardown_done_rx) = mpsc::channel::<()>();
let teardown = std::thread::spawn(move || {
drop(subscription);
teardown_done_tx.send(()).expect("signal teardown done");
});
assert!(
teardown_done_rx
.recv_timeout(Duration::from_millis(300))
.is_err(),
"teardown must block while a callback is in-flight",
);
release_tx.send(()).expect("release callback");
teardown_done_rx
.recv_timeout(Duration::from_secs(2))
.expect("teardown completes after the callback drains");
fire.join().expect("fire thread");
teardown.join().expect("teardown thread");
assert_eq!(ran.load(Ordering::SeqCst), 1, "callback ran exactly once");
assert_eq!(
store.subscriber_count(),
0,
"the drained guard removed the subscriber",
);
}
#[test]
fn dropping_the_external_guard_breaks_a_store_capturing_cycle() {
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let weak = Arc::downgrade(&store);
let captured = Arc::clone(&store);
let sub = store.subscribe_floors_raised(move |_raised| {
let _keep = &captured;
});
drop(sub);
drop(store);
assert!(
weak.upgrade().is_none(),
"dropping the external guard must break the callback→store cycle so the store frees",
);
}
#[test]
fn a_callback_can_drop_its_own_guard_without_deadlock() {
use std::sync::mpsc;
use std::time::Duration;
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let slot: Arc<Mutex<Option<RaiseSubscription>>> = Arc::new(Mutex::new(None));
let slot_cb = Arc::clone(&slot);
let sub = store.subscribe_floors_raised(move |_raised| {
let _dropped = slot_cb.lock().take();
});
*slot.lock() = Some(sub);
let (done_tx, done_rx) = mpsc::channel::<()>();
let store_t = Arc::clone(&store);
let worker = std::thread::spawn(move || {
store_t
.apply_bundle(&bundle_with_floor(5))
.expect("apply 5");
done_tx.send(()).expect("signal done");
});
assert!(
done_rx.recv_timeout(Duration::from_secs(5)).is_ok(),
"a callback dropping its own guard must not deadlock",
);
worker.join().expect("worker joined");
assert_eq!(
store.subscriber_count(),
0,
"the self-drop removed the subscription",
);
}
#[test]
fn self_unsubscribe_does_not_wait_for_a_concurrent_foreign_callback() {
use std::sync::mpsc;
use std::time::Duration;
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let user_lock = Arc::new(Mutex::new(()));
let role = Arc::new(AtomicUsize::new(0));
let slot: Arc<Mutex<Option<RaiseSubscription>>> = Arc::new(Mutex::new(None));
let (a_holds_tx, a_holds_rx) = mpsc::channel::<()>();
let (b_entered_tx, b_entered_rx) = mpsc::channel::<()>();
let (proceed_a_tx, proceed_a_rx) = mpsc::channel::<()>();
let a_holds_tx = Mutex::new(a_holds_tx);
let b_entered_tx = Mutex::new(b_entered_tx);
let proceed_a_rx = Mutex::new(proceed_a_rx);
let user_lock_cb = Arc::clone(&user_lock);
let role_cb = Arc::clone(&role);
let slot_cb = Arc::clone(&slot);
let sub = store.subscribe_floors_raised(move |_raised| {
if role_cb.fetch_add(1, Ordering::SeqCst) == 0 {
let held = user_lock_cb.lock();
a_holds_tx
.lock()
.send(())
.expect("A announces it holds the lock");
proceed_a_rx.lock().recv().expect("A awaits go-ahead");
drop(slot_cb.lock().take()); drop(held); } else {
b_entered_tx.lock().send(()).expect("B announces entry");
let _held = user_lock_cb.lock();
}
});
*slot.lock() = Some(sub);
let (done_tx, done_rx) = mpsc::channel::<()>();
let store1 = Arc::clone(&store);
let done1 = done_tx.clone();
let t1 = std::thread::spawn(move || {
store1.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
done1.send(()).expect("t1 done");
});
a_holds_rx
.recv_timeout(Duration::from_secs(5))
.expect("A entered and holds the user lock");
let store2 = Arc::clone(&store);
let done2 = done_tx.clone();
let t2 = std::thread::spawn(move || {
store2.apply_bundle(&bundle_with_floor(6)).expect("apply 6");
done2.send(()).expect("t2 done");
});
b_entered_rx
.recv_timeout(Duration::from_secs(5))
.expect("B entered the callback");
proceed_a_tx.send(()).expect("release A");
for _ in 0..2 {
done_rx
.recv_timeout(Duration::from_secs(5))
.expect("a worker deadlocked in self-unsubscribe");
}
t1.join().expect("thread 1 joined");
t2.join().expect("thread 2 joined");
assert_eq!(
role.load(Ordering::SeqCst),
2,
"exactly A and B ran (each once)",
);
assert_eq!(
store.subscriber_count(),
0,
"the self-drop removed the subscription",
);
store.apply_bundle(&bundle_with_floor(7)).expect("apply 7");
assert_eq!(
role.load(Ordering::SeqCst),
2,
"no callback runs after self-unsubscription removed the subscriber",
);
}
#[test]
fn aliased_paths_share_one_core() {
let scratch = Scratch::new();
let sub = scratch.0.join("sub");
std::fs::create_dir_all(&sub).expect("mkdir sub");
let direct = sub.join("revocation-state.json");
let dotted = scratch.0.join("sub/../sub/revocation-state.json");
let store_a = OrgRevocationStore::init(&direct, ProvisioningExpectation::MayBeFresh)
.expect("init direct");
let store_b = OrgRevocationStore::open_existing(&dotted).expect("open dotted alias");
assert!(
store_a.shares_core_with(&store_b),
"`..` alias joins the core"
);
store_a.apply_bundle(&bundle_with_floor(5)).expect("apply");
assert_eq!(store_b.floor_for(&org().org_id(), &member()), 5);
#[cfg(unix)]
{
let link = scratch.0.join("linked-sub");
std::os::unix::fs::symlink(&sub, &link).expect("symlink dir");
let via_link = OrgRevocationStore::open_existing(link.join("revocation-state.json"))
.expect("open through symlinked parent");
assert!(
store_a.shares_core_with(&via_link),
"symlinked-parent alias joins the core"
);
}
let bare = normalize_backing_path(Path::new("bare-floors.json")).expect("bare");
let dot = normalize_backing_path(Path::new("./bare-floors.json")).expect("dot");
assert!(bare.is_absolute());
assert_eq!(bare, dot);
assert!(
normalize_backing_path(&scratch.0.join("no-such-dir/state.json")).is_err(),
"unresolvable parent must refuse"
);
assert!(
normalize_backing_path(Path::new("..")).is_err(),
"no final component must refuse"
);
}
#[cfg(unix)]
#[test]
fn final_component_symlink_is_refused_atomically() {
let scratch = Scratch::new();
let path = scratch.state_path();
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
drop(OrgRevocationStore::open_existing(&path).expect("regular final opens"));
let real = scratch.0.join("real-state.json");
std::fs::rename(&path, &real).expect("move real");
std::os::unix::fs::symlink(&real, &path).expect("plant final symlink");
assert!(
normalize_backing_path(&path).is_err(),
"a symlink final component must refuse atomically"
);
assert!(
OrgRevocationStore::open_existing(&path).is_err(),
"open must refuse a symlink final"
);
}
#[cfg(unix)]
#[test]
fn non_regular_lock_sidecar_is_refused() {
let scratch = Scratch::new();
let path = scratch.state_path();
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let status = std::process::Command::new("mkfifo")
.arg(&lock_path)
.status()
.expect("run mkfifo");
assert!(status.success(), "mkfifo failed");
let err = OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh)
.expect_err("FIFO lock must refuse");
assert!(matches!(err, OrgRevocationError::Io { .. }), "got: {err}");
}
#[cfg(any(unix, windows))]
#[test]
fn hard_linked_lock_sidecar_is_refused() {
let scratch = Scratch::new();
let path = scratch.state_path();
drop(OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init"));
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock_path = PathBuf::from(lock_path);
let alias = scratch.0.join("alias.lock");
std::fs::hard_link(&lock_path, &alias).expect("hard-link the sidecar");
let err = OrgRevocationStore::open_existing(&path)
.expect_err("a hard-linked sidecar must be refused");
assert!(
matches!(&err, OrgRevocationError::Io { reason, .. } if reason.contains("hard links")),
"got: {err}",
);
}
#[test]
fn path_fallback_backing_id_retains_the_full_path() {
let a = BackingId::Path(PathBuf::from("/x/alpha/revocation-state.json"));
let a2 = BackingId::Path(PathBuf::from("/x/alpha/revocation-state.json"));
let b = BackingId::Path(PathBuf::from("/x/beta/revocation-state.json"));
assert_eq!(a, a2, "the same normalized path is the same identity");
assert_ne!(a, b, "distinct paths never share a fallback identity");
assert_ne!(
BackingId::FileId {
device: 0,
inode: 0
},
BackingId::Path(PathBuf::new()),
"file-identity and path-fallback are distinct identity spaces",
);
}
#[cfg(unix)]
#[test]
fn recreated_sidecar_under_a_live_core_is_refused() {
let scratch = Scratch::new();
let path = scratch.state_path();
let live =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock_path = PathBuf::from(lock_path);
let pin = std::fs::File::open(&lock_path).expect("pin the original sidecar inode");
std::fs::remove_file(&lock_path).expect("unlink the old sidecar");
let err = OrgRevocationStore::open_existing(&path)
.expect_err("a recreated sidecar under a live core must be refused");
assert!(
matches!(err, OrgRevocationError::BackingIdentityConflict { .. }),
"got: {err}",
);
drop(pin);
drop(live);
}
#[test]
fn raise_callback_fires_only_on_raises() {
let scratch = Scratch::new();
let store =
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init");
let seen: Arc<Mutex<Vec<RaisedFloor>>> = Arc::new(Mutex::new(Vec::new()));
let sink = seen.clone();
let _sub = store.subscribe_floors_raised(move |raised| {
sink.lock().extend_from_slice(raised);
});
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
assert_eq!(*seen.lock(), vec![(org().org_id(), member(), 5)]);
seen.lock().clear();
store.apply_bundle(&bundle_with_floor(3)).expect("apply 3");
assert!(seen.lock().is_empty(), "lower bundle must not notify");
}
#[cfg(unix)]
#[test]
fn post_rename_fsync_failure_poisons_the_path_until_recovery() {
use std::os::unix::fs::PermissionsExt;
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
let sibling = OrgRevocationStore::open_existing(&path).expect("sibling");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o300))
.expect("chmod 0300");
let err = store
.apply_bundle(&bundle_with_floor(9))
.expect_err("dir fsync must fail");
assert!(
matches!(err, OrgRevocationError::DurabilityUncertain { .. }),
"got: {err}"
);
assert_eq!(store.floor_for(&org().org_id(), &member()), 9);
assert!(store.is_poisoned());
assert!(
sibling.is_poisoned(),
"poison is path-wide, not per instance"
);
let err = store
.apply_bundle(&bundle_with_floor(11))
.expect_err("originating store refuses while recovery fails");
assert!(matches!(err, OrgRevocationError::Poisoned { .. }));
let err = sibling
.apply_bundle(&bundle_with_floor(3))
.expect_err("sibling refuses while the path is uncertain");
assert!(matches!(err, OrgRevocationError::Poisoned { .. }));
assert!(
OrgRevocationStore::open_existing(&path).is_err(),
"fresh open must not bypass path poison while recovery fails"
);
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o700))
.expect("chmod back");
let raised = sibling
.apply_bundle(&bundle_with_floor(11))
.expect("recovered apply succeeds");
assert!(raised.contains(&(org().org_id(), member(), 11)));
assert!(!store.is_poisoned(), "recovery clears the path-wide bit");
let reopened = OrgRevocationStore::open_existing(&path).expect("reopen");
assert_eq!(reopened.floor_for(&org().org_id(), &member()), 11);
}
#[cfg(unix)]
#[test]
fn poison_survives_dead_core_sidecar_recreation() {
use std::os::unix::fs::PermissionsExt;
let scratch = Scratch::new();
let path = scratch.state_path();
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock_path = PathBuf::from(lock_path);
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o300))
.expect("chmod 0300");
let err = store
.apply_bundle(&bundle_with_floor(9))
.expect_err("dir fsync must fail");
assert!(
matches!(err, OrgRevocationError::DurabilityUncertain { .. }),
"got: {err}"
);
assert!(store.is_poisoned());
drop(store);
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o700))
.expect("chmod 0700");
std::fs::remove_file(&lock_path).expect("unlink old sidecar");
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o300))
.expect("chmod 0300 again");
assert!(
OrgRevocationStore::open_existing(&path).is_err(),
"recovery must still be mandatory after sidecar recreation — poison survived",
);
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o700))
.expect("chmod back");
let recovered = OrgRevocationStore::open_existing(&path).expect("recovered reopen");
assert!(
!recovered.is_poisoned(),
"successful recovery clears poison"
);
assert_eq!(recovered.floor_for(&org().org_id(), &member()), 9);
let reopened = OrgRevocationStore::open_existing(&path).expect("clean reopen");
assert!(
!reopened.is_poisoned(),
"poison stays cleared (cleared exactly once)"
);
}
#[cfg(unix)]
#[test]
fn poison_survives_sidecar_recreation_across_a_cased_alias() {
use std::os::unix::fs::PermissionsExt;
let scratch = Scratch::new();
let lower = scratch.0.join("revocation-state.json");
let upper = scratch.0.join("REVOCATION-STATE.JSON");
let store = OrgRevocationStore::init(&lower, ProvisioningExpectation::MayBeFresh)
.expect("init lower");
if !upper.exists() {
return;
}
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o300))
.expect("chmod 0300");
let err = store
.apply_bundle(&bundle_with_floor(9))
.expect_err("dir fsync must fail");
assert!(matches!(
err,
OrgRevocationError::DurabilityUncertain { .. }
));
drop(store);
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o700))
.expect("chmod 0700");
let mut lock_path = lower.as_os_str().to_os_string();
lock_path.push(".lock");
std::fs::remove_file(PathBuf::from(lock_path)).expect("unlink sidecar");
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o300))
.expect("chmod 0300 again");
assert!(
OrgRevocationStore::open_existing(&upper).is_err(),
"poison must survive a sidecar swap AND a cased-alias reopen",
);
std::fs::set_permissions(&scratch.0, std::fs::Permissions::from_mode(0o700))
.expect("chmod back");
let recovered = OrgRevocationStore::open_existing(&upper).expect("recovered via alias");
assert!(!recovered.is_poisoned());
}
#[test]
fn clear_poison_retires_all_stale_ids_for_the_path() {
let scratch = Scratch::new();
let path = scratch.state_path();
let old_id = BackingId::FileId {
device: 0x5005,
inode: 0xF00D_0001,
};
let new_id = BackingId::FileId {
device: 0x5005,
inode: 0xF00D_0002,
};
mark_poisoned(&old_id, &path);
mark_poisoned(&new_id, &path);
assert!(is_poisoned(&old_id, &path));
assert!(is_poisoned(&new_id, &path));
clear_poison(&new_id, &path);
assert!(!is_poisoned(&new_id, &path), "recovered id cleared");
assert!(
!is_poisoned(&old_id, &path),
"stale old id retired with the path recovery — no dead residue",
);
}
#[cfg(unix)]
#[test]
fn generic_store_init_does_not_chmod_the_parent() {
use std::os::unix::fs::PermissionsExt;
let scratch = Scratch::new();
let parent = scratch.0.join("shared-app-dir");
std::fs::create_dir_all(&parent).expect("mkdir parent");
std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o755))
.expect("chmod 0755");
let store = OrgRevocationStore::init(
parent.join("revocation-state.json"),
ProvisioningExpectation::MayBeFresh,
)
.expect("init");
drop(store);
let mode = std::fs::metadata(&parent)
.expect("metadata")
.permissions()
.mode();
assert_eq!(
mode & 0o777,
0o755,
"generic store init must not chmod the parent (mode {mode:o})",
);
}
#[cfg(unix)]
#[test]
fn existing_handle_refuses_a_replaced_sidecar() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock_path = PathBuf::from(lock_path);
let pin = std::fs::File::open(&lock_path).expect("pin the original sidecar inode");
std::fs::remove_file(&lock_path).expect("unlink sidecar");
let err = store
.apply_bundle(&bundle_with_floor(9))
.expect_err("existing handle must refuse a replaced sidecar");
assert!(
matches!(err, OrgRevocationError::BackingIdentityConflict { .. }),
"got: {err}"
);
assert_eq!(store.floor_for(&org().org_id(), &member()), 5);
drop(pin);
drop(store);
let reopened = OrgRevocationStore::open_existing(&path).expect("reopen after drop");
assert_eq!(
reopened.floor_for(&org().org_id(), &member()),
5,
"the refused transaction must not have written floor 9 to disk",
);
}
#[test]
fn reentrant_callback_does_not_deadlock() {
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let reentered = Arc::new(Mutex::new(false));
let store_for_callback = Arc::downgrade(&store);
let flag = reentered.clone();
let _sub = store.subscribe_floors_raised(move |raised| {
if raised.iter().any(|(_, _, floor)| *floor == 5) {
if let Some(store) = store_for_callback.upgrade() {
store
.apply_bundle(&bundle_with_floor(7))
.expect("re-entrant apply must not deadlock");
*flag.lock() = true;
}
}
});
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
assert!(*reentered.lock(), "callback re-entered apply_bundle");
assert_eq!(store.floor_for(&org().org_id(), &member()), 7);
}
#[cfg(unix)]
#[test]
fn symlinked_state_and_lock_files_are_refused() {
let scratch = Scratch::new();
let path = scratch.state_path();
let store =
OrgRevocationStore::init(&path, ProvisioningExpectation::MayBeFresh).expect("init");
store.apply_bundle(&bundle_with_floor(5)).expect("apply 5");
drop(store);
let real = scratch.0.join("elsewhere.json");
std::fs::rename(&path, &real).expect("move state");
std::os::unix::fs::symlink(&real, &path).expect("plant symlink");
assert!(
OrgRevocationStore::open_existing(&path).is_err(),
"symlinked state file must refuse"
);
std::fs::remove_file(&path).expect("remove link");
std::fs::rename(&real, &path).expect("restore state");
OrgRevocationStore::open_existing(&path).expect("regular file opens");
let store = OrgRevocationStore::open_existing(&path).expect("open before planting");
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
let lock_path = PathBuf::from(lock_path);
let _ = std::fs::remove_file(&lock_path);
let foreign = scratch.0.join("foreign.lock");
std::fs::write(&foreign, b"").expect("foreign lock");
std::os::unix::fs::symlink(&foreign, &lock_path).expect("plant lock symlink");
assert!(
OrgRevocationStore::open_existing(&path).is_err(),
"symlinked lock sidecar must refuse the open"
);
assert!(
store.apply_bundle(&bundle_with_floor(9)).is_err(),
"symlinked lock sidecar must refuse a reload"
);
}
#[test]
fn barriered_generation_never_observes_an_in_progress_publish() {
use std::sync::mpsc;
let scratch = Scratch::new();
let store = Arc::new(
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init"),
);
let g0 = store.barriered_generation().expect("not exhausted").get();
let (swapped_rx, resume_tx) = store.arm_publish_pause_for_test();
let publisher = {
let store = store.clone();
std::thread::spawn(move || {
store.apply_bundle(&bundle_with_floor(9)).expect("apply");
})
};
swapped_rx.recv().expect("publisher reached the pause");
assert_eq!(
store.publish_generation(),
g0,
"bare read observes the pre-bump generation while the new floor is already swapped in",
);
let (reader_tx, reader_rx) = mpsc::channel();
let reader = {
let store = store.clone();
std::thread::spawn(move || {
let g = store.barriered_generation().expect("not exhausted");
let _ = reader_tx.send(g.get());
})
};
std::thread::sleep(std::time::Duration::from_millis(50));
assert!(
reader_rx.try_recv().is_err(),
"barriered read must block while the publish holds live.write() mid-swap",
);
resume_tx.send(()).expect("resume");
publisher.join().expect("publisher join");
reader.join().expect("reader join");
let observed = reader_rx.recv().expect("barriered read result");
assert_eq!(
observed,
g0 + 1,
"the barriered read returned the NEW generation, never the stale one",
);
assert_eq!(
store.barriered_generation().expect("not exhausted").get(),
g0 + 1
);
assert!(store.floor_for(&org().org_id(), &member()) >= 9);
}
#[test]
fn an_exhausted_publication_generation_freezes_rather_than_wrapping() {
let scratch = Scratch::new();
let store =
OrgRevocationStore::init(scratch.state_path(), ProvisioningExpectation::MayBeFresh)
.expect("init");
assert!(store.barriered_generation().is_ok());
store.saturate_generation_for_test();
assert_eq!(
store
.barriered_generation()
.expect("not yet exhausted")
.get(),
u64::MAX
);
store.republish_for_test();
assert_eq!(
store.barriered_generation(),
Err(GenerationExhausted),
"the exhausted space must be reported as an ERROR the caller cannot ignore, not as a frozen integer that reads as unchanged"
);
assert_eq!(
store.snapshot_with_generation().err(),
Some(GenerationExhausted),
"the coherent snapshot sampler fails closed the same way"
);
assert!(store.generation_exhausted_for_metrics());
store.republish_for_test();
assert_eq!(store.barriered_generation(), Err(GenerationExhausted));
}
}