use super::*;
use crate::adapter::net::behavior::org_grant::CapabilityAuthorityId;
use crate::adapter::net::behavior::org_routing_registry::{
GrantArtifactFence, GrantMovementFence, PrivateAudienceScope, ScopedDiscoveryAuthorityStamp,
ScopedSourceFacts, SlotKey, SlotSource, SourceCommitPin, SourceFacts, SourceSnapshot,
SourceToken,
};
use crate::adapter::net::behavior::org_scoped_ingest::CapabilityAudienceScope;
use crate::adapter::net::behavior::org_scoped_store::{
PrivateCapabilityProvider, PrivateDiscoveryDrains, PrivateDiscoveryStream,
};
async fn node() -> Arc<MeshNode> {
let addr: SocketAddr = "127.0.0.1:0".parse().expect("addr");
let cfg = MeshNodeConfig::new(addr, [0x77u8; 32]);
Arc::new(
MeshNode::new(EntityKeypair::generate(), cfg)
.await
.expect("MeshNode::new"),
)
}
fn owner_scope(seed: u8) -> PrivateAudienceScope {
PrivateAudienceScope::new(CapabilityAudienceScope::Owner {
org_id: crate::adapter::net::behavior::org::OrgId::from_bytes([seed; 32]),
audience_handle: [seed; 32],
})
.expect("owner scopes are private")
}
fn slot(seed: u8, tag: &str) -> SlotKey {
SlotKey {
scope: owner_scope(seed),
capability: CapabilityAuthorityId::for_tag(tag),
}
}
async fn until(mut f: impl FnMut() -> bool) -> bool {
for _ in 0..2000 {
if f() {
return true;
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
false
}
async fn quiesced_counts(node: &MeshNode) -> [u64; 7] {
for _ in 0..100 {
let before = node.org_routing_reconciliation_counts();
for _ in 0..2 {
let passes = node.org_routing_actor_passes();
node.mark_org_routing_work();
assert!(
until(|| node.org_routing_actor_passes() > passes).await,
"the routing actor never completed a pass, so no exact delta is \
measurable"
);
}
let (_retained, pending) = node.org_routing_slots();
if pending == 0 && node.org_routing_reconciliation_counts() == before {
return before;
}
}
panic!("the routing actor never quiesced, so no exact delta is measurable");
}
#[tokio::test]
async fn startup_mints_exactly_one_global_drain_and_leaves_owner_unclaimed() {
let node = node().await;
{
let probe = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
probe.mint(PrivateDiscoveryStream::Global).is_some(),
"an unstarted node holds no global lease"
);
}
node.start();
assert!(
until(|| node.routing_task.lock().is_some()).await,
"the supervisor task must be recorded for joining"
);
assert!(
until(|| node.org_routing_supervision_counts().0 >= 1).await,
"an incarnation must start"
);
{
let rival = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_none(),
"the running actor holds the exclusive global drain"
);
assert!(
rival.mint(PrivateDiscoveryStream::Owner).is_some(),
"the owner stream stays unclaimed for the leader track"
);
}
assert!(until(|| node.org_routing_ready()).await, "healthy first");
let started = node.org_routing_supervision_counts().0;
node.start_org_routing_supervisor();
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(
node.org_routing_supervision_counts().0,
started,
"there is exactly one production supervisor construction path"
);
assert!(
node.org_routing_ready(),
"a refused duplicate start must not fence the live routing plane"
);
let _ = node.shutdown().await;
assert!(
node.routing_task.lock().is_none(),
"the handle shutdown joined was the LIVE one, not a duplicate's"
);
}
#[tokio::test]
async fn the_dirty_stream_reaches_the_real_registry() {
let node = node().await;
node.start();
assert!(
until(|| node.org_routing_ready()).await,
"the mint's RebuildAll must complete a recapture and publish Healthy"
);
let family = node.org_routing_family().expect("family");
let key = slot(1, "nrpc:e3c");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| node.org_routing_slots() == (1, 0)).await,
"the actor must reconcile the demanded slot, not merely count wakes"
);
let facts = node
.org_routing_base_facts(&key)
.expect("rebuilt through the production source");
assert_eq!(
facts.epoch.generation,
node.scoped_discovery.lock().revision(),
"facts carry the query-visible generation they were committed against"
);
assert!(
node.org_routing_reconciliation_counts()[0] >= 1,
"a real installation happened"
);
assert_eq!(node.org_routing_unserved_scope_count(), 0);
drop(held);
let _ = node.shutdown().await;
}
#[tokio::test]
async fn shutdown_joins_the_routing_task_before_returning() {
let node = node().await;
node.start();
assert!(until(|| node.org_routing_ready()).await, "healthy");
{
let rival = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_none(),
"the lease is held while the actor runs"
);
}
let _ = node.shutdown().await;
assert!(
node.routing_task.lock().is_none(),
"shutdown must consume the routing task handle"
);
assert!(
!node.org_routing_ready(),
"the actor fenced routing health on its way out"
);
let rival = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_some(),
"the drain dropped BEFORE shutdown returned, so a successor can mint"
);
}
type BuildHook = Box<dyn Fn() + Send + Sync>;
struct PausingSource {
inner: ScopedSlotSource,
during_build: Arc<parking_lot::Mutex<Option<BuildHook>>>,
after_pin: Arc<parking_lot::Mutex<Option<BuildHook>>>,
before_pin: Arc<parking_lot::Mutex<Option<BuildHook>>>,
}
struct PausingSnapshot {
inner: Box<dyn SourceSnapshot>,
during_build: Arc<parking_lot::Mutex<Option<BuildHook>>>,
}
impl SourceSnapshot for PausingSnapshot {
fn token(&self) -> SourceToken {
self.inner.token()
}
fn providers(&self, key: &SlotKey) -> ScopedSourceFacts {
let hook = self.during_build.lock().take();
if let Some(hook) = hook {
hook();
}
self.inner.providers(key)
}
}
impl SlotSource for PausingSource {
fn snapshot(&self, keys: &[SlotKey]) -> Box<dyn SourceSnapshot> {
Box::new(PausingSnapshot {
inner: self.inner.snapshot(keys),
during_build: self.during_build.clone(),
})
}
fn pin_if_current(
&self,
_keys: &[SlotKey],
expected: &SourceToken,
) -> Option<Box<dyn SourceCommitPin + '_>> {
let hook = self.before_pin.lock().take();
if let Some(hook) = hook {
hook();
}
let pin = self.inner.pin_if_current(_keys, expected)?;
let hook = self.after_pin.lock().take();
if let Some(hook) = hook {
hook();
}
Some(pin)
}
fn liveness(&self) -> crate::adapter::net::behavior::org_routing_registry::SourceLiveness {
self.inner.liveness()
}
}
#[tokio::test]
async fn a_mutation_during_reconstruction_defeats_the_production_commit_pin() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scoped = node.scoped_discovery.clone();
let publication = node.scoped_publication.clone();
let during_build: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(
Arc::new(PausingSource {
inner: ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: scoped.clone(),
publication: publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
},
during_build: during_build.clone(),
after_pin: Arc::default(),
before_pin: Arc::default(),
}),
work.clone(),
Arc::default(),
);
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(2, "nrpc:pinned");
let _held = family.demand(key.clone()).expect("demand");
let landed = Arc::new(AtomicBool::new(false));
{
let scoped = scoped.clone();
let publication = publication.clone();
let landed = landed.clone();
*during_build.lock() = Some(Box::new(move || {
publication.gated_commit(&scoped, |s| {
s.advance_query_visible_generation_for_test(CapabilityAuthorityId::for_tag(
"nrpc:other",
))
});
landed.store(true, Ordering::Release);
}));
}
let before = scoped.lock().revision();
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: before,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
landed.load(Ordering::Acquire),
"the rival mutation must have completed DURING reconstruction"
);
assert!(
scoped.lock().revision() > before,
"and must have advanced the query-visible generation"
);
assert_eq!(
outcome,
ApplyOutcome::Superseded,
"the stale snapshot's commit pin must refuse"
);
assert!(
registry.base_facts_unvalidated(&key).is_none(),
"nothing from the stale snapshot installed"
);
assert_eq!(
registry.pending_slots(),
1,
"the exact selected identity was requeued"
);
let current = scoped.lock().revision();
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: current,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the successor pass completes"
);
assert_eq!(
registry
.base_facts_unvalidated(&key)
.expect("installed")
.epoch
.generation,
current,
"at the CURRENT generation"
);
}
#[tokio::test]
async fn the_production_source_is_scope_exact_and_counts_unserved_scopes() {
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let grant_key = SlotKey {
scope: PrivateAudienceScope::new(CapabilityAudienceScope::Grant {
grant_id: [9u8; 32],
audience_handle: [9u8; 32],
})
.expect("grant scopes are private"),
capability: CapabilityAuthorityId::for_tag("nrpc:g"),
};
let owner_key = slot(3, "nrpc:o");
let snapshot = source.snapshot(&[owner_key.clone(), grant_key.clone()]);
assert!(
matches!(snapshot.providers(&owner_key).facts, SourceFacts::Served(ref p) if p.is_empty()),
"an owner scope with no rows is SERVED with exact empty evidence"
);
assert!(
matches!(snapshot.providers(&grant_key).facts, SourceFacts::Unserved),
"an unsupported scope is UNSERVED, not authoritatively empty"
);
assert_eq!(
node.org_routing_unserved_scope_count(),
1,
"the unserved grant scope is COUNTED, not silently empty"
);
let token = snapshot.token();
assert!(
source
.pin_if_current(&[], &SourceToken::new(vec![u64::MAX]))
.is_none(),
"a token the source has left is refused"
);
assert!(
source.pin_if_current(&[], &token).is_some(),
"the live token is accepted while the snapshot is still held"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn shutdown_overlapping_registration_cannot_return_unresolved() {
let node = node().await;
let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel::<()>(1);
let release = Arc::new((parking_lot::Mutex::new(false), parking_lot::Condvar::new()));
{
let release = release.clone();
*node.routing_spawn_pause_hook.lock() = Some(Arc::new(move || {
let _ = entered_tx.try_send(());
let (lock, cv) = &*release;
let mut go = lock.lock();
while !*go {
cv.wait(&mut go);
}
}));
}
let starter = {
let node = node.clone();
tokio::task::spawn_blocking(move || node.start_org_routing_supervisor())
};
tokio::task::spawn_blocking(move || entered_rx.recv())
.await
.expect("join")
.expect("startup must reach the spawn/publication window");
let shutdown_returned = Arc::new(AtomicBool::new(false));
let shutting = {
let node = node.clone();
let flag = shutdown_returned.clone();
tokio::spawn(async move {
let _ = node.shutdown().await;
flag.store(true, Ordering::Release);
})
};
let blocked = {
let node = node.clone();
tokio::task::spawn_blocking(move || {
for _ in 0..20_000 {
if node.routing_join_blocked_for_test() {
return true;
}
std::thread::sleep(Duration::from_millis(1));
}
false
})
}
.await
.expect("join");
assert!(blocked, "shutdown never reached the routing-task slot");
assert!(
!shutdown_returned.load(Ordering::Acquire),
"shutdown returned while routing registration was still unresolved"
);
{
let (lock, cv) = &*release;
*lock.lock() = true;
cv.notify_all();
}
starter.await.expect("starter");
shutting.await.expect("shutdown task");
assert!(shutdown_returned.load(Ordering::Acquire));
assert!(
node.routing_task.lock().is_none(),
"shutdown took and joined the exact handle"
);
assert!(!node.org_routing_ready(), "health is fenced");
let rival = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_some(),
"no post-shutdown incarnation survives holding the drain"
);
}
#[tokio::test]
async fn startup_that_loses_to_shutdown_spawns_nothing() {
let node = node().await;
node.shutdown_flag_for_test();
node.start_org_routing_supervisor();
assert!(
node.routing_task.lock().is_none(),
"startup that observes shutdown under the slot lock must spawn nothing"
);
let rival = PrivateDiscoveryDrains::new(node.scoped_discovery.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_some(),
"and must not have claimed the exclusive drain"
);
}
#[tokio::test]
async fn revocation_authority_movement_alone_defeats_the_commit_pin() {
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(4, "nrpc:floored");
let snapshot = source.snapshot(std::slice::from_ref(&key));
let token = snapshot.token();
let scoped_before = node.scoped_discovery.lock().revision();
assert!(
source.pin_if_current(&[], &token).is_some(),
"nothing has moved yet"
);
let scratch = std::env::temp_dir().join(format!(
"net-olb2b-e3c-rev-{}-{}",
std::process::id(),
node.entity_id()
));
let _ = std::fs::remove_dir_all(&scratch);
std::fs::create_dir_all(&scratch).expect("scratch dir");
let store = Arc::new(
crate::adapter::net::behavior::org_revocation::OrgRevocationStore::init(
scratch.join("revocation.json"),
crate::adapter::net::behavior::org_revocation::ProvisioningExpectation::MayBeFresh,
)
.expect("init revocation store"),
);
node.install_org_revocation_store(store.clone())
.expect("install revocation store");
assert_eq!(
node.scoped_discovery.lock().revision(),
scoped_before,
"the scoped revision must NOT move - only revocation authority did"
);
assert!(
source.pin_if_current(&[], &token).is_none(),
"a snapshot taken under the OLD revocation authority cannot commit"
);
let fresh = source.snapshot(std::slice::from_ref(&key));
let fresh_token = fresh.token();
assert!(source.pin_if_current(&[], &fresh_token).is_some());
store.mark_poisoned_for_test();
assert!(
source.pin_if_current(&[], &fresh_token).is_none(),
"poisoning the revocation authority invalidates a snapshot taken before it"
);
let poisoned = source.snapshot(std::slice::from_ref(&key));
assert!(
matches!(poisoned.providers(&key).facts, SourceFacts::Unserved),
"a poisoned revocation authority serves NOTHING rather than unfiltered rows"
);
drop(store);
node.org_revocation.store(None);
let _ = std::fs::remove_dir_all(&scratch);
}
#[tokio::test]
async fn an_unserved_scope_reads_cold_rather_than_authoritatively_empty() {
let node = node().await;
node.start();
assert!(until(|| node.org_routing_ready()).await, "healthy");
let family = node.org_routing_family().expect("family");
let grant_key = SlotKey {
scope: PrivateAudienceScope::new(CapabilityAudienceScope::Grant {
grant_id: [4u8; 32],
audience_handle: [4u8; 32],
})
.expect("grant scopes are private"),
capability: CapabilityAuthorityId::for_tag("nrpc:granted"),
};
let owner_key = slot(5, "nrpc:owned");
let _g = family.demand(grant_key.clone()).expect("grant demand");
let _o = family.demand(owner_key.clone()).expect("owner demand");
assert!(
until(|| node.org_routing_slots() == (2, 0)).await,
"both slots reconcile - an unserved scope still owes no work"
);
assert!(
node.org_routing_base_facts(&grant_key).is_none(),
"the unserved grant scope reads COLD, not as zero providers"
);
assert!(
node.org_routing_base_facts(&owner_key).is_some(),
"the served owner scope reads as real evidence"
);
assert!(node.org_routing_unserved_scope_count() >= 1);
let _ = node.shutdown().await;
}
#[tokio::test]
async fn cached_facts_that_crossed_their_expiry_read_cold() {
use crate::adapter::net::behavior::org_routing_registry::{SlotBaseFacts, SourceFacts};
let node = node().await;
let key = slot(6, "nrpc:expiring");
let now = crate::adapter::net::behavior::org::current_timestamp();
let expired = Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: crate::adapter::net::behavior::org_routing_registry::SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: now,
});
let live = Arc::new(SlotBaseFacts {
earliest_expiry: now + 3600,
..(*expired).clone()
});
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
node.routing_registry
.install_facts_for_test(key.clone(), live);
assert!(
node.org_routing_base_facts(&key).is_some(),
"an unexpired cached fact is served"
);
node.routing_registry
.install_facts_for_test(key.clone(), expired);
assert!(
node.org_routing_base_facts(&key).is_none(),
"an expired cached fact must NOT be served merely because the exact \
timer has not swept yet"
);
}
#[tokio::test]
async fn drop_fences_and_aborts_rather_than_detaching_the_supervisor() {
let scoped = {
let node = node().await;
node.start();
assert!(until(|| node.org_routing_ready()).await, "healthy");
let scoped = node.scoped_discovery.clone();
{
let rival = PrivateDiscoveryDrains::new(scoped.clone());
assert!(
rival.mint(PrivateDiscoveryStream::Global).is_none(),
"the lease is held while the actor runs"
);
}
let health = node.routing_health.clone();
drop(node);
assert!(
matches!(
**health.load(),
crate::adapter::net::behavior::org_routing::RoutingHealth::Fenced
),
"Drop must fence routing health SYNCHRONOUSLY"
);
scoped
};
assert!(
until(|| {
PrivateDiscoveryDrains::new(scoped.clone())
.mint(PrivateDiscoveryStream::Global)
.is_some()
})
.await,
"an aborted supervisor must release the exclusive global drain"
);
}
struct Scratch(std::path::PathBuf);
impl Scratch {
fn path(&self) -> &std::path::Path {
&self.0
}
fn new(tag: &str, node: &MeshNode) -> Self {
static SEQ: AtomicU64 = AtomicU64::new(0);
let seq = SEQ.fetch_add(1, Ordering::Relaxed);
let entity = format!("{}", node.entity_id());
let path = std::env::temp_dir().join(format!(
"olb-{tag}-{}-{seq}-{}",
std::process::id(),
&entity[..entity.len().min(8)]
));
let _ = std::fs::remove_dir_all(&path);
std::fs::create_dir_all(&path).expect("scratch dir");
Self(path)
}
fn store(&self) -> Arc<crate::adapter::net::behavior::org_revocation::OrgRevocationStore> {
let store = Arc::new(
crate::adapter::net::behavior::org_revocation::OrgRevocationStore::init(
self.0.join("revocation.json"),
crate::adapter::net::behavior::org_revocation::ProvisioningExpectation::MayBeFresh,
)
.expect("init revocation store"),
);
let generation = store.barriered_generation().expect(
"a freshly created store cannot be generation-exhausted; if it is, this \
store JOINED another test's live core through a recycled sidecar inode",
);
assert_eq!(
generation.get(),
0,
"a freshly created store must be at generation 0; a nonzero one means this store \
joined another test's live core through a recycled sidecar inode"
);
assert!(
!store.is_poisoned(),
"a freshly created store cannot be poisoned; a poisoned one means this store \
joined another test's live core through a recycled sidecar inode"
);
store
}
}
fn arm_authority_contention(node: &MeshNode) -> std::sync::mpsc::Receiver<()> {
let (tx, rx) = std::sync::mpsc::sync_channel::<()>(4);
*node.routing_authority.contention_hook.lock() = Some(Arc::new(move || {
let _ = tx.try_send(());
}));
rx
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_production_store_swap_cannot_publish_while_a_commit_pin_is_alive() {
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(7, "nrpc:pinned-authority");
let scratch = Scratch::new("swap", &node);
let store = scratch.store();
let blocked = arm_authority_contention(&node);
let snapshot = source.snapshot(std::slice::from_ref(&key));
let pin = source
.pin_if_current(&[], &snapshot.token())
.expect("nothing has moved");
let installed = Arc::new(AtomicBool::new(false));
let rival = {
let node = node.clone();
let installed = installed.clone();
let store = store.clone();
tokio::task::spawn_blocking(move || {
node.install_org_revocation_store(store)
.expect("install revocation store");
installed.store(true, Ordering::Release);
})
};
tokio::task::spawn_blocking(move || blocked.recv_timeout(Duration::from_secs(10)))
.await
.expect("join")
.expect("the installer must block on the authority gate");
assert!(
!installed.load(Ordering::Acquire),
"the production install completed while a commit pin was alive"
);
assert!(
node.org_revocation.load().is_none(),
"and the new store must NOT be query-visible yet"
);
drop(pin);
rival.await.expect("rival");
assert!(installed.load(Ordering::Acquire));
assert!(node.org_revocation.load().is_some());
assert!(
source.pin_if_current(&[], &snapshot.token()).is_none(),
"a token from the retired authority must be refused"
);
}
#[tokio::test]
async fn facts_built_against_superseded_floors_read_cold() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
let scratch = Scratch::new("floors", &node);
node.install_org_revocation_store(scratch.store())
.expect("install");
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let key = slot(11, "nrpc:floored-read");
let live_floor = node
.org_revocation
.load()
.as_ref()
.expect("store")
.barriered_generation()
.expect("not exhausted")
.get();
node.routing_registry.install_facts_for_test(
key.clone(),
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: live_floor,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
}),
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: coherent facts are served"
);
node.routing_registry.install_facts_for_test(
key.clone(),
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: live_floor.wrapping_sub(1),
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
}),
);
assert!(
node.org_routing_base_facts(&key).is_none(),
"facts built against superseded floors must read COLD even though the \
authority epoch and scoped revision are unchanged"
);
assert_eq!(node.org_routing_slots().1, 1, "and the slot is re-queued");
}
#[tokio::test]
async fn an_exhausted_authority_epoch_fences_rather_than_aliasing() {
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(12, "nrpc:exhausted");
node.routing_authority
.epoch
.store(u64::MAX, Ordering::Release);
assert!(!node.routing_authority.is_exhausted());
{
let _gate = node.routing_authority.lock_gate();
assert_eq!(
node.routing_authority.advance(),
AuthorityAdvance::NewlyExhausted,
"the terminal transition is reported to exactly one caller"
);
assert_eq!(
node.routing_authority.advance(),
AuthorityAdvance::AlreadyExhausted,
"and never a second time"
);
}
assert!(
node.routing_authority.is_exhausted(),
"advancing past the ceiling must FENCE, not saturate"
);
assert_eq!(
node.routing_authority.epoch(),
u64::MAX,
"and must not have handed out a reused identity"
);
let snapshot = source.snapshot(std::slice::from_ref(&key));
assert!(
matches!(snapshot.providers(&key).facts, SourceFacts::Unserved),
"an exhausted authority serves nothing"
);
assert!(
source.pin_if_current(&[], &snapshot.token()).is_none(),
"and commits nothing"
);
}
#[tokio::test]
async fn an_exhausted_store_generation_makes_every_scope_unserved() {
let node = node().await;
let scratch = Scratch::new("gen-exhausted", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(14, "nrpc:gen-exhausted");
let healthy = source.snapshot(std::slice::from_ref(&key));
assert!(
matches!(healthy.providers(&key).facts, SourceFacts::Served(_)),
"precondition: a usable authority serves the scope"
);
let healthy_token = healthy.token();
assert!(source.pin_if_current(&[], &healthy_token).is_some());
store.saturate_generation_for_test();
store.republish_for_test();
assert_eq!(
store.barriered_generation().err(),
Some(crate::adapter::net::behavior::org_revocation::GenerationExhausted)
);
let snapshot = source.snapshot(std::slice::from_ref(&key));
assert!(
matches!(snapshot.providers(&key).facts, SourceFacts::Unserved),
"an exhausted publication generation serves NOTHING"
);
assert!(
source.pin_if_current(&[], &healthy_token).is_none(),
"and a token minted under the usable authority no longer commits"
);
}
#[tokio::test]
async fn terminal_exhaustion_retires_max_stamped_facts_and_fences_readiness() {
use crate::adapter::net::behavior::org_routing::{ApplyOutcome, ApplyRequest, DirtyApply};
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("authority-terminal", &node);
node.install_org_revocation_store(scratch.store())
.expect("install");
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let registry = node.routing_registry.clone();
registry.activate_incarnation(1);
node.routing_authority
.epoch
.store(u64::MAX, Ordering::Release);
let family = registry.new_family().expect("family");
let key = slot(26, "nrpc:authority-terminal");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
let warm = registry.base_facts_unvalidated(&key).expect("reconciled");
assert_eq!(
warm.epoch.authority,
u64::MAX,
"precondition: MAX-stamped facts are retained"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: and warm"
);
assert!(node.org_routing_ready(), "precondition: ready");
let replacement = Scratch::new("authority-terminal-b", &node);
node.install_org_revocation_store(replacement.store())
.expect("replacement install");
assert!(node.routing_authority.is_exhausted(), "terminally fenced");
assert_eq!(
node.routing_authority.epoch(),
u64::MAX,
"no identity was reused"
);
assert!(
registry.base_facts_unvalidated(&key).is_none(),
"the MAX-stamped fact was retired SYNCHRONOUSLY — no reader involved"
);
assert_eq!(
registry.pending_slots(),
0,
"and nothing was re-queued: a rebuild could never install again"
);
assert!(
matches!(
**node.routing_health.load(),
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { .. }
),
"supervisor health alone still says Healthy…"
);
assert!(
!node.org_routing_ready(),
"…so readiness must fence on exhaustion INDEPENDENTLY of health"
);
assert!(
node.org_routing_base_facts(&key).is_none(),
"reads are cold"
);
let late = slot(26, "nrpc:authority-terminal-late");
let _late_held = family.demand(late.clone()).expect("late demand");
assert_eq!(registry.pending_slots(), 1, "the late demand queues once");
assert_eq!(
registry.apply(1, request),
ApplyOutcome::Superseded,
"nothing settles under a terminal authority"
);
assert_eq!(
registry.pending_slots(),
0,
"and the pass DISCARDS rather than re-queues — no terminal spin"
);
assert!(registry.base_facts_unvalidated(&late).is_none());
}
#[tokio::test]
async fn an_exhausted_store_generation_parks_apply_without_spinning_and_recovers() {
use crate::adapter::net::behavior::org_routing::{ApplyOutcome, ApplyRequest, DirtyApply};
use crate::adapter::net::behavior::org_routing_registry::SourceLiveness;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("store-generation-fence", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let registry = node.routing_registry.clone();
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(27, "nrpc:store-generation-fence");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: the slot is warm through the production seam"
);
store.saturate_generation_for_test();
store.republish_for_test();
assert_eq!(
store.barriered_generation().err(),
Some(crate::adapter::net::behavior::org_revocation::GenerationExhausted)
);
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
assert_eq!(
source.liveness(),
SourceLiveness::Fenced,
"recoverable, not terminal: a replacement install retires it"
);
assert!(
!node.routing_authority.is_exhausted(),
"and the AUTHORITY latch is untouched — this is the other arm"
);
registry.invalidate_for_test(&key);
node.routing_work.take_for_test();
for pass in 0..3 {
assert_eq!(
registry.apply(1, request.clone()),
ApplyOutcome::Superseded,
"pass {pass}: nothing settles against an exhausted publication generation"
);
assert_eq!(
registry.pending_slots(),
1,
"pass {pass}: the identity stays OWED — the fence is recoverable"
);
assert!(
!node.routing_work.take_for_test(),
"pass {pass}: and the pass must NOT re-arm itself — that is the livelock"
);
}
assert!(
node.org_routing_base_facts(&key).is_none(),
"service is cold while the fence holds"
);
let replacement = Scratch::new("store-generation-fence-b", &node);
node.install_org_revocation_store(replacement.store())
.expect("replacement install");
assert_eq!(
source.liveness(),
SourceLiveness::Live,
"the fence lifts with the store that raised it"
);
assert!(
node.routing_work.take_for_test(),
"and the movement supplied the wake the parked pass deliberately did not"
);
assert!(
matches!(
registry.apply(1, request),
ApplyOutcome::Current { .. } | ApplyOutcome::Progress { .. }
),
"the preserved queue is what lets the successor pass rebuild"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"and service is warm again — the fence cost promptness, not the slot"
);
}
#[tokio::test]
async fn an_empty_registry_whose_authority_moves_under_the_probe_is_redriven() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("empty-selection", &node);
node.install_org_revocation_store(scratch.store())
.expect("install");
let before_pin: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let after_pin: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(
Arc::new(PausingSource {
inner: ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
},
during_build: Arc::default(),
after_pin: after_pin.clone(),
before_pin: before_pin.clone(),
}),
work.clone(),
Arc::default(),
);
registry.activate_incarnation(1);
assert_eq!(registry.retained_slots(), 0);
assert_eq!(registry.pending_slots(), 0);
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(
matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
),
"baseline: an undisturbed empty pass settles Current"
);
let replacement = Scratch::new("empty-selection-b", &node);
let replacement_store = replacement.store();
let moved = Arc::new(AtomicBool::new(false));
{
let node = node.clone();
let replacement_store = replacement_store.clone();
let moved = moved.clone();
*before_pin.lock() = Some(Box::new(move || {
node.install_org_revocation_store(replacement_store.clone())
.expect("replacement install");
moved.store(true, Ordering::Release);
}));
}
work.take_for_test();
assert_eq!(
registry.apply(1, request.clone()),
ApplyOutcome::Superseded,
"the pin must refuse a token minted under the retired authority"
);
assert!(moved.load(Ordering::Acquire), "the authority did move");
assert!(
work.take_for_test(),
"the empty-selection PIN refusal must mark: nothing else will wake this actor"
);
assert!(
matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
),
"and the re-driven pass settles — which is what lets the supervisor publish Healthy"
);
let published = Arc::new(AtomicBool::new(false));
{
let replacement_store = replacement_store.clone();
let published = published.clone();
*after_pin.lock() = Some(Box::new(move || {
replacement_store.republish_for_test();
published.store(true, Ordering::Release);
}));
}
work.take_for_test();
assert_eq!(
registry.apply(1, request.clone()),
ApplyOutcome::Superseded,
"a floor publication under the pin must not settle an empty pass either"
);
assert!(published.load(Ordering::Acquire), "the floor did move");
assert!(
work.take_for_test(),
"the empty-selection SETTLE refusal must mark for the same reason"
);
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
replacement_store.saturate_generation_for_test();
replacement_store.republish_for_test();
work.take_for_test();
for pass in 0..3 {
assert_eq!(
registry.apply(1, request.clone()),
ApplyOutcome::Superseded,
"pass {pass}: an exhausted publication generation cannot settle"
);
assert!(
!work.take_for_test(),
"pass {pass}: and must NOT self-wake — the empty path spins hardest of all"
);
}
}
#[tokio::test]
async fn an_exhausted_scoped_generation_fences_the_routing_source() {
use crate::adapter::net::behavior::org_routing_registry::SourceLiveness;
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(31, "nrpc:generation-ceiling");
let healthy = source.snapshot(std::slice::from_ref(&key));
let healthy_token = healthy.token();
assert!(
matches!(healthy.providers(&key).facts, SourceFacts::Served(_)),
"precondition: a usable generation serves the scope"
);
drop(healthy);
assert_eq!(source.liveness(), SourceLiveness::Live);
node.scoped_publication
.gated_commit(&node.scoped_discovery, |state| {
state.park_revisions_at_ceiling_for_test();
state.advance_query_visible_generation_for_test(CapabilityAuthorityId::for_tag(
"nrpc:generation-ceiling",
));
});
assert!(
node.scoped_discovery.lock().generations_exhausted(),
"the advance past the ceiling LATCHES rather than wrapping to 0"
);
assert_eq!(
node.scoped_discovery.lock().revision(),
u64::MAX,
"and parks on the terminal sentinel"
);
assert_eq!(
source.liveness(),
SourceLiveness::Terminal,
"a generation that can no longer distinguish states is terminal authority"
);
let fenced = source.snapshot(std::slice::from_ref(&key));
assert!(
matches!(fenced.providers(&key).facts, SourceFacts::Unserved),
"an exhausted change generation serves NOTHING"
);
let fenced_token = fenced.token();
drop(fenced);
assert!(
source.pin_if_current(&[], &healthy_token).is_none(),
"a token minted under the usable generation no longer commits"
);
assert!(
source.pin_if_current(&[], &fenced_token).is_none(),
"and neither does one minted UNDER the exhaustion — two exhausted samples \
must never compare equal-and-current"
);
}
#[tokio::test]
async fn terminal_exhaustion_is_visible_on_a_metrics_surface() {
let node = node().await;
let scratch = Scratch::new("exhaustion-surface", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
assert_eq!(
node.org_authority_exhaustion(),
(false, false, false),
"precondition: nothing is exhausted"
);
store.saturate_generation_for_test();
store.republish_for_test();
assert!(
node.org_authority_exhaustion().0,
"the revocation publication generation latch is readable after the fact"
);
node.routing_authority
.exhausted
.store(true, Ordering::Release);
node.scoped_publication
.gated_commit(&node.scoped_discovery, |state| {
state.park_revisions_at_ceiling_for_test();
state.advance_query_visible_generation_for_test(CapabilityAuthorityId::for_tag(
"nrpc:exhaustion-surface",
));
});
assert_eq!(
node.org_authority_exhaustion(),
(true, true, true),
"and each plane reports its OWN latch — they are separate identity spaces"
);
}
#[tokio::test]
async fn start_publishes_every_background_handle_before_returning() {
let node = node().await;
assert!(
node.tasks.lock().is_empty(),
"precondition: an unstarted node owns no background tasks"
);
node.start();
let published = node.tasks.lock().len();
assert!(
published >= 10,
"every background handle must be joinable the instant `start` returns, \
with no scheduler round-trip in between (published {published})"
);
let _ = node.shutdown().await;
assert!(
node.tasks.lock().is_empty(),
"and shutdown consumed exactly the handles start published"
);
}
#[tokio::test]
async fn a_delayed_reader_does_not_delete_a_newer_artifact() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
let key = slot(13, "nrpc:delayed-reader");
let facts = |authority: u64| {
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority,
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
})
};
let stale = facts(node.routing_authority.epoch().wrapping_sub(1));
node.routing_registry
.install_facts_for_test(key.clone(), stale.clone());
let current = facts(node.routing_authority.epoch());
node.routing_registry
.install_facts_for_test(key.clone(), current.clone());
node.routing_registry.invalidate_if_stale(&key, &stale);
let live = node
.routing_registry
.base_facts_unvalidated(&key)
.expect("the current artifact survives");
assert!(
Arc::ptr_eq(&live, ¤t),
"a delayed reader must not delete a newer artifact"
);
assert_eq!(
node.org_routing_slots().1,
0,
"and must not re-queue work that was already done"
);
}
#[tokio::test]
async fn the_read_seam_fences_a_dead_incarnations_facts() {
use crate::adapter::net::behavior::org_routing::RoutingHealth;
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
let key = slot(29, "nrpc:incarnation-fence");
let facts = Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
});
node.routing_registry
.install_facts_for_test(key.clone(), facts.clone());
node.routing_health
.store(Arc::new(RoutingHealth::Healthy { incarnation: 1 }));
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: under its OWN live incarnation the artifact serves"
);
for (health, why) in [
(
RoutingHealth::Fenced,
"a fenced plane must serve nothing, however current the artifact is",
),
(
RoutingHealth::Rebuilding { incarnation: 1 },
"nor may a rebuilding one serve what its own incarnation already built",
),
(
RoutingHealth::Healthy { incarnation: 2 },
"and a SUCCESSOR incarnation does not inherit its predecessor's artifacts",
),
] {
node.routing_health.store(Arc::new(health));
assert!(node.org_routing_base_facts(&key).is_none(), "{why}");
assert!(
node.routing_registry.base_facts_unvalidated(&key).is_some(),
"…and the raw accessor still returns it, which is why the SEAM has to fence"
);
}
}
#[tokio::test]
async fn the_read_seams_authority_sample_cannot_straddle_a_store_install() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
assert!(node.org_revocation.load().is_none());
let key = slot(30, "nrpc:coherent-sample");
let facts = Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
});
node.routing_registry
.install_facts_for_test(key.clone(), facts.clone());
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: warm under the pre-install authority"
);
let scratch = Scratch::new("coherent-sample", &node);
let landed = Arc::new(AtomicBool::new(false));
{
let inner = node.clone();
let store = scratch.store();
let landed = landed.clone();
*node.routing_sample_gap_hook.lock() = Some(Arc::new(move || {
inner
.install_org_revocation_store(store.clone())
.expect("install");
landed.store(true, Ordering::Release);
}));
}
let served = node.org_routing_base_facts(&key);
assert!(landed.load(Ordering::Acquire), "the install did land");
assert!(
served.is_none(),
"a sample straddling the install must never serve A-era facts as B-authoritative"
);
assert_ne!(
facts.epoch.authority,
node.routing_authority.epoch(),
"and the movement really did change the identity the facts were stamped with"
);
}
#[tokio::test]
async fn poisoning_authority_colds_already_cached_facts() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
let key = slot(8, "nrpc:poisoned");
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let scratch = std::env::temp_dir().join(format!(
"net-olb2b-e3c-poison-{}-{}",
std::process::id(),
node.entity_id()
));
let _ = std::fs::remove_dir_all(&scratch);
std::fs::create_dir_all(&scratch).expect("scratch dir");
let store = Arc::new(
crate::adapter::net::behavior::org_revocation::OrgRevocationStore::init(
scratch.join("revocation.json"),
crate::adapter::net::behavior::org_revocation::ProvisioningExpectation::MayBeFresh,
)
.expect("init revocation store"),
);
node.org_revocation.store(Some(store.clone()));
node.routing_registry.install_facts_for_test(
key.clone(),
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
}),
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: the cached facts are served"
);
store.mark_poisoned_for_test();
assert!(
node.org_routing_base_facts(&key).is_none(),
"poisoned authority must COLD already-cached facts"
);
assert_eq!(
node.org_routing_slots().1,
1,
"and re-queue the exact slot so the actor rebuilds it"
);
node.org_revocation.store(None);
drop(store);
let _ = std::fs::remove_dir_all(&scratch);
}
#[tokio::test]
async fn authority_only_movement_invalidates_and_requeues_everything() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let key = slot(9, "nrpc:authority-move");
let scoped_before = node.scoped_discovery.lock().revision();
node.routing_registry.install_facts_for_test(
key.clone(),
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: scoped_before,
authority: node.routing_authority.epoch(),
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
}),
);
assert!(node.org_routing_base_facts(&key).is_some());
move_routing_authority(&node.routing_authority, &node.routing_registry, || {});
assert_eq!(
node.scoped_discovery.lock().revision(),
scoped_before,
"authority movement touched NO scoped state - there is no scoped wake"
);
assert!(
node.routing_registry.base_facts_unvalidated(&key).is_none(),
"authority movement must SYNCHRONOUSLY invalidate every retained fact"
);
assert_eq!(
node.org_routing_slots().1,
1,
"and re-queue it, so the actor rebuilds without waiting for a reader"
);
assert!(
node.org_routing_base_facts(&key).is_none(),
"and it reads cold"
);
}
#[tokio::test]
async fn a_recapture_across_two_authority_epochs_does_not_settle_current() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(
Arc::new(ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
}),
work.clone(),
Arc::default(),
);
registry.activate_incarnation(1);
let mut family = registry.new_family().expect("family");
let mut held = Vec::new();
let mut keys = Vec::new();
for index in 0..70 {
if index > 0 && index % 64 == 0 {
family = registry.new_family().expect("family");
}
let key = slot(10, &format!("nrpc:epoch-{index}"));
held.push(family.demand(key.clone()).expect("demand"));
keys.push(key);
}
let request = |dirty| ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty,
},
registry_work: true,
};
let outcome = registry.apply(1, request(DirtyCapabilities::RebuildAll));
assert!(
matches!(outcome, ApplyOutcome::Progress { .. }),
"one quantum cannot finish 70 slots: {outcome:?}"
);
let authority_a = node.routing_authority.epoch();
let scoped = node.scoped_discovery.lock().revision();
{
let _gate = node.routing_authority.lock_gate();
let _ = node.routing_authority.advance();
}
assert_eq!(
node.scoped_discovery.lock().revision(),
scoped,
"scoped revision is IDENTICAL across the two authority epochs"
);
assert_ne!(node.routing_authority.epoch(), authority_a);
let outcome = registry.apply(1, request(DirtyCapabilities::RebuildAll));
assert!(
matches!(outcome, ApplyOutcome::Progress { .. }),
"a recapture must not settle Current over mixed-authority facts: {outcome:?}"
);
assert!(
registry.pending_slots() > 0,
"the authority-A slots are re-queued"
);
let mut outcome = registry.apply(1, request(DirtyCapabilities::RebuildAll));
for _ in 0..4 {
if matches!(outcome, ApplyOutcome::Current { .. }) {
break;
}
outcome = registry.apply(1, request(DirtyCapabilities::RebuildAll));
}
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"and completes once every slot shares one authority: {outcome:?}"
);
let live = node.routing_authority.epoch();
for key in &keys {
assert_eq!(
registry
.base_facts_unvalidated(key)
.expect("built")
.epoch
.authority,
live,
"one authority across the whole retained set"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_snapshot_cannot_straddle_a_store_publication() {
let node = node().await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = slot(15, "nrpc:straddle");
let scratch = Scratch::new("straddle", &node);
let blocked = arm_authority_contention(&node);
let (release_tx, release_rx) = std::sync::mpsc::sync_channel::<()>(1);
let (done_tx, done_rx) = std::sync::mpsc::sync_channel::<()>(1);
let epoch_before = node.routing_authority.epoch();
let holder = {
let node = node.clone();
let store = scratch.store();
tokio::task::spawn_blocking(move || {
let _held = node.routing_authority.lock_gate();
let _ = done_tx.send(());
let _ = release_rx.recv();
let _ = node.routing_authority.advance();
node.org_revocation.store(Some(store));
})
};
tokio::task::spawn_blocking(move || done_rx.recv_timeout(Duration::from_secs(10)))
.await
.expect("join")
.expect("the holder must take the gate");
let sampled = Arc::new(AtomicBool::new(false));
let sampler = {
let sampled = sampled.clone();
let source_epoch = Arc::new(parking_lot::Mutex::new(None));
let out = source_epoch.clone();
let node = node.clone();
let key = key.clone();
(
tokio::task::spawn_blocking(move || {
let src = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let snap = src.snapshot(std::slice::from_ref(&key));
*out.lock() = Some(snap.token());
sampled.store(true, Ordering::Release);
}),
source_epoch,
)
};
tokio::task::spawn_blocking(move || blocked.recv_timeout(Duration::from_secs(10)))
.await
.expect("join")
.expect("the snapshot must block on the authority gate");
assert!(
!sampled.load(Ordering::Acquire),
"a snapshot sampled authority while a publication held the gate"
);
let _ = release_tx.send(());
holder.await.expect("holder");
assert_ne!(epoch_before, node.routing_authority.epoch());
sampler.0.await.expect("sampler");
let token = sampler.1.lock().clone().expect("token");
assert!(
source.pin_if_current(&[], &token).is_some(),
"the snapshot must have sampled one side of the transition, not a mix"
);
}
#[tokio::test]
async fn the_epoch_advances_before_the_store_becomes_visible() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let key = slot(16, "nrpc:epoch-first");
let retired = node.routing_authority.epoch();
node.routing_registry.install_facts_for_test(
key.clone(),
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: retired,
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
}),
);
assert!(node.org_routing_base_facts(&key).is_some());
let scratch = Scratch::new("epoch-first", &node);
let store = scratch.store();
let observed = Arc::new(AtomicBool::new(false));
{
let node2 = node.clone();
let observed = observed.clone();
let store = store.clone();
move_routing_authority(&node.routing_authority, &node.routing_registry, || {
assert_ne!(
node2.routing_authority.epoch(),
retired,
"the epoch must advance BEFORE the store becomes visible"
);
assert!(
node2.org_revocation.load().is_none(),
"precondition: the store is not visible yet"
);
node2.org_revocation.store(Some(store));
observed.store(true, Ordering::Release);
});
}
assert!(observed.load(Ordering::Acquire));
assert!(
node.org_routing_base_facts(&key).is_none(),
"facts stamped with the retired epoch read cold across the transition"
);
}
#[tokio::test]
async fn a_floor_publication_under_the_pin_cannot_settle_current() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("pin-floor", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
let during_build: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let after_pin: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let work: Arc<RegistryWork> = Arc::default();
let metrics: Arc<crate::adapter::net::behavior::org_routing_registry::RegistryMetrics> =
Arc::default();
let registry = NodeOrgRoutingRegistry::new(
Arc::new(PausingSource {
inner: ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
},
during_build: during_build.clone(),
after_pin: after_pin.clone(),
before_pin: Arc::default(),
}),
work.clone(),
metrics.clone(),
);
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(17, "nrpc:pin-floor");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
registry.invalidate_for_test(&key);
let published = Arc::new(AtomicBool::new(false));
{
let store = store.clone();
let published = published.clone();
*after_pin.lock() = Some(Box::new(move || {
store.republish_for_test();
published.store(true, Ordering::Release);
}));
}
let outcome = registry.apply(1, request.clone());
assert!(
published.load(Ordering::Acquire),
"the floor must have moved"
);
assert_eq!(
outcome,
ApplyOutcome::Superseded,
"a quantum whose floor authority moved must NOT settle Current"
);
assert_eq!(registry.pending_slots(), 1, "and must re-queue");
assert_eq!(
metrics.settlements_refused(),
1,
"the refusal is counted, and counted as a settlement refusal"
);
assert_eq!(
metrics.stale_actor_rejections(),
0,
"and NOT as actor-lifecycle churn"
);
assert!(matches!(
registry.apply(1, request),
ApplyOutcome::Current { .. }
));
}
#[tokio::test]
async fn authority_invalidation_spares_successor_facts() {
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let stale_key = slot(18, "nrpc:retired");
let fresh_key = slot(18, "nrpc:successor");
let retired = node.routing_authority.epoch();
{
let _gate = node.routing_authority.lock_gate();
let _ = node.routing_authority.advance();
}
let live = node.routing_authority.epoch();
assert_ne!(retired, live);
let facts = |authority: u64| {
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Served(Arc::from([] as [PrivateCapabilityProvider; 0])),
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority,
floor_generation: 0,
poisoned: false,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
})
};
node.routing_registry
.install_facts_for_test(stale_key.clone(), facts(retired));
let successor = facts(live);
node.routing_registry
.install_facts_for_test(fresh_key.clone(), successor.clone());
node.routing_registry.invalidate_authority_older_than(live);
assert!(
node.routing_registry
.base_facts_unvalidated(&stale_key)
.is_none(),
"the retired-authority facts are invalidated"
);
let survivor = node
.routing_registry
.base_facts_unvalidated(&fresh_key)
.expect("successor facts survive");
assert!(
Arc::ptr_eq(&survivor, &successor),
"invalidation must not delete work done under the SUCCESSOR authority"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_publication_cannot_occupy_the_gap_between_validation_and_settlement() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("settle-gap", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
};
let published = Arc::new(AtomicBool::new(false));
let entered = Arc::new(AtomicBool::new(false));
let publisher: Arc<parking_lot::Mutex<Option<std::thread::JoinHandle<()>>>> = Arc::default();
let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel::<()>(1);
let reached = Arc::new(parking_lot::Mutex::new(reached_rx));
let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
store.arm_publish_contended_hook(Arc::new(move || {
let _ = reached_tx.try_send(());
}));
{
let store = store.clone();
let published = published.clone();
let entered = entered.clone();
let publisher = publisher.clone();
let reached = reached.clone();
*source.settle_gap_hook.lock() = Some(Arc::new(move || {
entered.store(true, Ordering::Release);
let store = store.clone();
let landed = published.clone();
let done = done_tx.clone();
*publisher.lock() = Some(std::thread::spawn(move || {
store.republish_for_test();
landed.store(true, Ordering::Release);
let _ = done.send(());
}));
reached.lock().recv_timeout(Duration::from_secs(10)).expect(
"the publisher's try_write must FAIL, proving the settlement pin \
holds the publication barrier; no signal means the barrier was \
released before the settlement",
);
assert!(
!published.load(Ordering::Acquire),
"a floor publication landed between the validation and the \
settlement — they are separately interleavable"
);
}));
}
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(Arc::new(source), work.clone(), Arc::default());
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(19, "nrpc:settle-gap");
let _held = family.demand(key.clone()).expect("demand");
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
entered.load(Ordering::Acquire),
"the gap must have been entered"
);
done_rx
.recv_timeout(Duration::from_secs(10))
.expect("the publication must land once the barrier is released");
if let Some(handle) = publisher.lock().take() {
handle.join().expect("publisher");
}
assert!(
published.load(Ordering::Acquire),
"the publication must land once the barrier is released"
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the settlement was protected, so it is sound: {outcome:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn poison_cannot_occupy_the_gap_between_validation_and_settlement() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("settle-poison", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
};
let poisoned = Arc::new(AtomicBool::new(false));
let entered = Arc::new(AtomicBool::new(false));
let contender: Arc<parking_lot::Mutex<Option<std::thread::JoinHandle<()>>>> = Arc::default();
let (at_gate_tx, at_gate_rx) = std::sync::mpsc::sync_channel::<()>(1);
let at_gate = Arc::new(parking_lot::Mutex::new(at_gate_rx));
store.arm_poison_contended_hook(Arc::new(move || {
let _ = at_gate_tx.try_send(());
}));
let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
let done_rx = Arc::new(parking_lot::Mutex::new(done_rx));
{
let store = store.clone();
let poisoned = poisoned.clone();
let entered = entered.clone();
let contender = contender.clone();
let at_gate = at_gate.clone();
let done_rx = done_rx.clone();
*source.settle_gap_hook.lock() = Some(Arc::new(move || {
entered.store(true, Ordering::Release);
let contender_store = store.clone();
let landed = poisoned.clone();
let done = done_tx.clone();
*contender.lock() = Some(std::thread::spawn(move || {
contender_store.mark_poisoned_for_test();
landed.store(true, Ordering::Release);
let _ = done.send(());
}));
at_gate
.lock()
.recv_timeout(Duration::from_secs(10))
.expect("the contender must OBSERVE the poison gate held");
assert!(
done_rx.lock().try_recv().is_err(),
"poison landed between the validation and the settlement — they \
are separately interleavable"
);
assert!(
!poisoned.load(Ordering::Acquire),
"poison landed between the validation and the settlement — they \
are separately interleavable"
);
assert!(
!store.is_poisoned(),
"the poison FACT landed inside the validation-settlement gap — \
the mark mutated the registry before taking the gate"
);
}));
}
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(Arc::new(source), work.clone(), Arc::default());
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(20, "nrpc:settle-poison");
let _held = family.demand(key.clone()).expect("demand");
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
entered.load(Ordering::Acquire),
"the gap must have been entered"
);
done_rx
.lock()
.recv_timeout(Duration::from_secs(10))
.expect("poison must land once the pin is released");
if let Some(handle) = contender.lock().take() {
handle.join().expect("contender");
}
assert!(
poisoned.load(Ordering::Acquire),
"poison must land once the pin is released"
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the settlement was protected, so it is sound: {outcome:?}"
);
}
#[tokio::test]
async fn poison_before_the_validation_is_detected() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("poison-early", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
let after_pin: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(
Arc::new(PausingSource {
inner: ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
},
during_build: Arc::default(),
after_pin: after_pin.clone(),
before_pin: Arc::default(),
}),
work.clone(),
Arc::default(),
);
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(21, "nrpc:poison-early");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
registry.invalidate_for_test(&key);
let landed = Arc::new(AtomicBool::new(false));
{
let store = store.clone();
let landed = landed.clone();
*after_pin.lock() = Some(Box::new(move || {
store.mark_poisoned_for_test();
landed.store(true, Ordering::Release);
}));
}
let outcome = registry.apply(1, request);
assert!(landed.load(Ordering::Acquire), "poison must have landed");
assert_eq!(
outcome,
ApplyOutcome::Superseded,
"poison completing before the validation must NOT settle Current"
);
assert_eq!(registry.pending_slots(), 1, "and must re-queue");
}
#[tokio::test]
async fn steady_poison_settles_current_over_an_unserved_source() {
use crate::adapter::net::behavior::org_routing::{ApplyOutcome, ApplyRequest, DirtyApply};
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("poison-steady", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
store.mark_poisoned_for_test();
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let registry = node.routing_registry.clone();
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(22, "nrpc:poison-steady");
let _held = family.demand(key.clone()).expect("demand");
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"steady poison must CONVERGE, not retry forever: {outcome:?}"
);
assert_eq!(registry.pending_slots(), 0, "nothing is owed");
let facts = registry
.base_facts_unvalidated(&key)
.expect("the slot IS reconciled — with unusable-source facts");
assert!(
matches!(
facts.providers,
crate::adapter::net::behavior::org_routing_registry::SourceFacts::Unserved
),
"a poisoned authority can speak for no scope"
);
assert!(
facts.epoch.poisoned,
"and the facts are STAMPED with the poisoned authority, so a later \
recovery is detectable"
);
assert!(
node.org_routing_base_facts(&key).is_none(),
"but it reads COLD: reconciled is not usable"
);
assert_eq!(
registry.pending_slots(),
0,
"and a cold read under STEADY poison re-queues nothing — the epoch \
comparison catches transitions, not the steady state"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_recovery_poison_clear_cannot_land_under_a_publication_pin() {
use crate::adapter::net::behavior::org_revocation::OrgRevocationStore;
let node = node().await;
let scratch = Scratch::new("poison-clear", &node);
let store = scratch.store();
store.mark_poisoned_for_test();
assert!(store.is_poisoned(), "precondition: the path is poisoned");
let (at_gate_tx, at_gate_rx) = std::sync::mpsc::sync_channel::<()>(1);
let (pinned_tx, pinned_rx) = std::sync::mpsc::channel::<()>();
let pinned_rx = parking_lot::Mutex::new(pinned_rx);
store.arm_poison_blocking_hook(Arc::new(move || {
let _ = at_gate_tx.try_send(());
let _ = pinned_rx.lock().recv_timeout(Duration::from_secs(10));
}));
let (contended_tx, contended_rx) = std::sync::mpsc::sync_channel::<()>(1);
store.arm_poison_contended_hook(Arc::new(move || {
let _ = contended_tx.try_send(());
}));
let (done_tx, done_rx) = std::sync::mpsc::channel::<bool>();
let path = scratch.path().join("revocation.json");
let recovery = std::thread::spawn(move || {
let recovered = OrgRevocationStore::open_existing(&path);
let _ = done_tx.send(recovered.is_ok());
});
at_gate_rx
.recv_timeout(Duration::from_secs(10))
.expect("the recovery must REACH the poison gate");
let pin = store.pin_publication();
let _ = pinned_tx.send(());
contended_rx
.recv_timeout(Duration::from_secs(10))
.expect("the recovery's clear must OBSERVE the pin's poison-gate hold");
assert!(
done_rx.try_recv().is_err(),
"a production recovery cleared poison while a publication pin was alive \
— the clear bypasses the poison gate"
);
assert!(
store.is_poisoned(),
"poison must be held immobile in BOTH directions under the pin"
);
drop(pin);
assert!(
done_rx
.recv_timeout(Duration::from_secs(10))
.expect("the clear must land once the pin drops"),
"the recovery itself must succeed"
);
assert!(!store.is_poisoned(), "recovery clears the poison");
recovery.join().expect("recovery thread");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_poison_recovery_retires_the_unserved_reconstruction_it_left() {
use crate::adapter::net::behavior::org_revocation::OrgRevocationStore;
use crate::adapter::net::behavior::org_routing::{ApplyOutcome, ApplyRequest, DirtyApply};
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("poison-recover", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
store.mark_poisoned_for_test();
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let registry = node.routing_registry.clone();
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(23, "nrpc:poison-recover");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(
matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
),
"steady poison converges (Option A)"
);
let stranded = registry.base_facts_unvalidated(&key).expect("reconciled");
assert!(
matches!(stranded.providers, SourceFacts::Unserved),
"precondition: the reconstruction serves nothing"
);
assert!(stranded.epoch.poisoned, "precondition: stamped as poisoned");
assert!(
node.org_routing_base_facts(&key).is_none(),
"precondition: it reads cold"
);
let authority_before = node.routing_authority.epoch();
let path = scratch.path().join("revocation.json");
let recovered = tokio::time::timeout(
Duration::from_secs(10),
tokio::task::spawn_blocking(move || OrgRevocationStore::open_existing(&path)),
)
.await
.expect("bounded: the recovery open must complete")
.expect("join")
.expect("recovery must succeed");
assert!(!store.is_poisoned(), "recovery clears the poison");
assert!(
recovered.shares_core_with(&store),
"and it recovers the SAME live core the node installed"
);
assert!(
node.routing_authority.epoch() > authority_before,
"a poison recovery must move routing authority even though it raised no \
floor"
);
assert!(
registry.base_facts_unvalidated(&key).is_none(),
"and it must RETIRE the Unserved reconstruction, not leave it reconciled"
);
assert_eq!(
registry.pending_slots(),
1,
"re-queuing the exact slot, so the actor rebuilds rather than leaving a \
hole"
);
assert!(
matches!(registry.apply(1, request), ApplyOutcome::Current { .. }),
"the successor quantum settles"
);
let successor = registry
.base_facts_unvalidated(&key)
.expect("reconciled again");
assert!(
!successor.epoch.poisoned,
"the successor is stamped against the RECOVERED authority"
);
assert!(
matches!(successor.providers, SourceFacts::Served(_)),
"and the source speaks for the scope again"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"so the slot reads warm — recovery is observable end to end"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_poison_mark_that_raises_no_floor_wakes_routing_without_a_reader() {
use crate::adapter::net::behavior::org::{OrgId, OrgKeypair, OrgRevocationBundle};
use crate::adapter::net::behavior::org_revocation::{OrgRevocationError, OrgRevocationStore};
use crate::adapter::net::behavior::org_routing::{ApplyOutcome, ApplyRequest, DirtyApply};
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("poison-mark", &node);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install");
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let registry = node.routing_registry.clone();
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(27, "nrpc:poison-mark");
let _held = family.demand(key.clone()).expect("demand");
let request = ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
};
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
assert!(
node.org_routing_base_facts(&key).is_some(),
"precondition: warm over a healthy authority"
);
let floor_org = OrgKeypair::from_bytes([0x42u8; 32]);
let floor_member = EntityId::from_bytes([0x24u8; 32]);
let org_id: OrgId = floor_org.org_id();
let bundle = |floor: u32| {
let mut floors = std::collections::BTreeMap::new();
floors.insert(floor_member.clone(), floor);
OrgRevocationBundle::try_issue(&floor_org, &floors).expect("issue")
};
let epoch_before = node.routing_authority.epoch();
let err = {
store.arm_forced_post_rename_for_test();
store
.apply_bundle(&bundle(5))
.expect_err("durability must be uncertain")
};
assert!(matches!(
err,
OrgRevocationError::DurabilityUncertain { .. }
));
assert!(store.is_poisoned(), "the raising mark poisoned the path");
assert_eq!(
node.routing_authority.epoch(),
epoch_before + 1,
"a RAISING mark wakes through `notify` — and exactly once"
);
assert_eq!(
store.floor_for(&org_id, &floor_member),
5,
"the live view is ahead of what disk can prove"
);
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
let path = scratch.path().join("revocation.json");
let recovered = tokio::time::timeout(
Duration::from_secs(10),
tokio::task::spawn_blocking(move || OrgRevocationStore::open_existing(&path)),
)
.await
.expect("bounded: the recovery open must complete")
.expect("join")
.expect("recovery must succeed");
assert!(recovered.shares_core_with(&store), "same live core");
assert!(!store.is_poisoned(), "recovery clears the poison");
assert_eq!(
store.floor_for(&org_id, &floor_member),
5,
"recovery rereads older disk bytes but never weakens the live view"
);
assert!(matches!(
registry.apply(1, request.clone()),
ApplyOutcome::Current { .. }
));
let warm = registry.base_facts_unvalidated(&key).expect("reconciled");
assert!(
!warm.epoch.poisoned,
"precondition: warm again, stamped against the recovered authority"
);
let authority_before = node.routing_authority.epoch();
let err = {
store.arm_forced_post_rename_for_test();
store
.apply_bundle(&bundle(3))
.expect_err("durability must be uncertain again")
};
assert!(matches!(
err,
OrgRevocationError::DurabilityUncertain { .. }
));
assert!(store.is_poisoned(), "the empty-raise mark landed");
assert_eq!(
store.floor_for(&org_id, &floor_member),
5,
"and raised no floor"
);
assert_eq!(
node.routing_authority.epoch(),
authority_before + 1,
"an empty-raise MARK owes the same wake as the recovery clear"
);
assert!(
registry.base_facts_unvalidated(&key).is_none(),
"the pre-poison reconstruction was RETIRED, not left reconciled"
);
assert_eq!(registry.pending_slots(), 1, "re-queuing the exact slot");
assert!(matches!(
registry.apply(1, request),
ApplyOutcome::Current { .. }
));
let successor = registry
.base_facts_unvalidated(&key)
.expect("reconciled again");
assert!(
matches!(successor.providers, SourceFacts::Unserved),
"Current over an Unserved source"
);
assert!(
successor.epoch.poisoned,
"stamped against the poisoned authority, so a later recovery is \
detectable"
);
assert!(
node.org_routing_base_facts(&key).is_none(),
"and reads cold"
);
}
#[tokio::test]
async fn a_reader_retires_unserved_facts_once_their_poison_clears() {
use crate::adapter::net::behavior::org_revocation::OrgRevocationStore;
use crate::adapter::net::behavior::org_routing_registry::{
SlotBaseFacts, SourceEpoch, SourceFacts,
};
let node = node().await;
node.routing_health.store(Arc::new(
crate::adapter::net::behavior::org_routing::RoutingHealth::Healthy { incarnation: 1 },
));
let scratch = Scratch::new("poison-lazy", &node);
let store = scratch.store();
node.org_revocation.store(Some(store.clone()));
store.mark_poisoned_for_test();
let poisoned_facts = |floor_generation: u64| {
Arc::new(SlotBaseFacts {
authority: ScopedDiscoveryAuthorityStamp::Owner,
grant_fence: GrantArtifactFence::Publication(0),
providers: SourceFacts::Unserved,
epoch: SourceEpoch {
generation: node.scoped_discovery.lock().revision(),
authority: node.routing_authority.epoch(),
floor_generation,
poisoned: true,
},
actor_incarnation: 1,
slot_incarnation: 1,
earliest_expiry: u64::MAX,
})
};
let key_a = slot(24, "nrpc:poison-lazy-steady");
let live_floor = store.barriered_generation().expect("not exhausted").get();
node.routing_registry
.install_facts_for_test(key_a.clone(), poisoned_facts(live_floor));
assert!(
node.org_routing_base_facts(&key_a).is_none(),
"an unusable source reads cold"
);
assert_eq!(
node.org_routing_slots().1,
0,
"and STEADY poison re-queues nothing: reading live poison as a staleness \
predicate would churn this slot on every read, forever"
);
let path = scratch.path().join("revocation.json");
let _recovered = tokio::time::timeout(
Duration::from_secs(10),
tokio::task::spawn_blocking(move || OrgRevocationStore::open_existing(&path)),
)
.await
.expect("bounded: the recovery open must complete")
.expect("join")
.expect("recovery must succeed");
assert!(!store.is_poisoned(), "recovery clears the poison");
assert!(
node.org_routing_base_facts(&key_a).is_none(),
"the obsolete reconstruction still reads cold"
);
assert!(
node.routing_registry
.base_facts_unvalidated(&key_a)
.is_none(),
"but the reader RETIRED it rather than leaving it reconciled forever"
);
assert_eq!(node.org_routing_slots().1, 1, "re-queuing the exact slot");
let key_b = slot(24, "nrpc:poison-lazy-isolated");
let recovered_floor = store.barriered_generation().expect("not exhausted").get();
node.routing_registry
.install_facts_for_test(key_b.clone(), poisoned_facts(recovered_floor));
assert!(node.org_routing_base_facts(&key_b).is_none());
assert!(
node.routing_registry
.base_facts_unvalidated(&key_b)
.is_none(),
"a fact differing from the live authority ONLY in the poison bit must \
still be retired"
);
assert_eq!(node.org_routing_slots().1, 2, "and re-queued");
node.org_revocation.store(None);
}
#[tokio::test]
async fn an_exhausted_store_generation_stops_certified_emission() {
let node = node().await;
let org = crate::adapter::net::behavior::org::OrgKeypair::generate();
let entity = node.entity_id().clone();
let cert = crate::adapter::net::behavior::org::OrgMembershipCert::try_issue(
&org,
entity.clone(),
1,
3600,
)
.expect("cert");
let scratch = Scratch::new("send-exhausted", &node);
let authority = crate::adapter::net::behavior::org_authority::NodeAuthority::adopt(
scratch.path(),
cert,
&entity,
0,
None,
)
.expect("adopt");
node.install_node_authority(Arc::new(authority))
.expect("install authority");
let _ = node.set_owner_cert_emission(true);
let store = scratch.store();
node.install_org_revocation_store(store.clone())
.expect("install store");
node.announce_capabilities(crate::adapter::net::behavior::capability::CapabilitySet::default())
.await
.expect("announce");
let now = crate::adapter::net::behavior::org::current_timestamp();
let live_authority = node.node_authority().expect("authority installed");
assert!(
node.owner_cert_under(&live_authority, now).is_some(),
"precondition: the send path would certify this announcement"
);
assert!(
node.announcement_bytes_for_send_for_test().is_some(),
"precondition: there is something to send"
);
store.saturate_generation_for_test();
store.republish_for_test();
assert!(
store.barriered_generation().is_err(),
"terminally exhausted"
);
assert!(
node.owner_cert_under(&live_authority, now).is_none(),
"terminal exhaustion must not construct an owner certificate — the floor comparison it rests on can no longer be shown current"
);
assert!(
node.announcement_scoped_for_send_for_test().is_empty(),
"terminal exhaustion must not emit owner/grant-scoped envelopes"
);
let exhausted_stamp = node.security_stamp();
assert!(
!exhausted_stamp.is_current(&exhausted_stamp),
"an exhausted send stamp must never compare current, even against itself"
);
}
#[tokio::test]
async fn duplicate_provider_rows_collapse_to_the_newest_generation() {
let node = node().await;
let key = slot(25, "nrpc:dedup");
let provider = node.entity_id().clone();
let org = crate::adapter::net::behavior::org::OrgId::from_bytes([25u8; 32]);
let row = |generation: u64, expires_at: u64| PrivateCapabilityProvider {
provider: provider.clone(),
owner_org: org,
expires_at,
generation,
};
let snapshot = ScopedSourceSnapshot {
token: SourceToken::default(),
grant_publication: 0,
grant_publications_spent: false,
rows: [(
key.clone(),
(
vec![row(3, 300), row(9, 900), row(5, 500)],
ScopedDiscoveryAuthorityStamp::Owner,
u64::MAX,
),
)]
.into_iter()
.collect(),
};
let SourceFacts::Served(providers) = snapshot.providers(&key).facts else {
panic!("a captured scope must reconstruct as Served");
};
assert_eq!(providers.len(), 1, "one row per provider survives");
assert_eq!(
providers[0].generation, 9,
"the dedup must keep the NEWEST announcement, not the oldest"
);
assert_eq!(
providers[0].expires_at, 900,
"and the surviving row must be that announcement's, not a mix"
);
}
fn adopt_authority(
node: &MeshNode,
org: &crate::adapter::net::behavior::org::OrgKeypair,
tag: &str,
) -> Arc<crate::adapter::net::behavior::org_authority::NodeAuthority> {
use crate::adapter::net::behavior::org::OrgMembershipCert;
use crate::adapter::net::behavior::org_authority::NodeAuthority;
let entity = node.entity_id().clone();
let cert = OrgMembershipCert::try_issue(org, entity.clone(), 1, 3600).expect("issue cert");
static SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let dir = std::env::temp_dir().join(format!("net-olb2c-{tag}-{}-{seq}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
Arc::new(NodeAuthority::adopt(&dir, cert, &entity, 0, None).expect("adopt authority"))
}
#[tokio::test]
async fn an_authority_install_publishes_the_authority_under_its_own_epoch() {
let node = node().await;
let org = crate::adapter::net::behavior::org::OrgKeypair::from_bytes([0x2cu8; 32]);
node.install_node_authority(adopt_authority(&node, &org, "a"))
.expect("install A");
let a = node.node_authority().expect("A is installed");
let next = adopt_authority(&node, &org, "b");
let early = Arc::new(AtomicBool::new(false));
{
let early = early.clone();
let premature = next.clone();
let weak = Arc::downgrade(&node);
*node.routing_authority.pre_publish_hook.lock() = Some(Arc::new(move |_epoch| {
if let Some(node) = weak.upgrade() {
if node
.node_authority()
.is_some_and(|live| Arc::ptr_eq(&live, &premature))
{
early.store(true, Ordering::Release);
}
}
}));
}
let observed: Arc<parking_lot::Mutex<Vec<(u64, bool)>>> =
Arc::new(parking_lot::Mutex::new(Vec::new()));
{
let sink = observed.clone();
let weak = Arc::downgrade(&node);
let expected = next.clone();
*node.routing_authority.post_publish_hook.lock() = Some(Arc::new(move |epoch| {
let Some(node) = weak.upgrade() else {
return;
};
let live_is_next = node
.node_authority()
.is_some_and(|live| Arc::ptr_eq(&live, &expected));
sink.lock().push((epoch, live_is_next));
}));
}
node.install_node_authority(next.clone())
.expect("install B");
*node.routing_authority.post_publish_hook.lock() = None;
*node.routing_authority.pre_publish_hook.lock() = None;
assert!(
!early.load(Ordering::Acquire),
"the replacement authority was already visible BEFORE the epoch advance — a reader in that window observes it under the OLD epoch identity"
);
let observed = observed.lock().clone();
assert_eq!(
observed.len(),
1,
"the complete authority+store transaction must publish under exactly ONE \
epoch advance, not one per half"
);
assert!(
observed[0].1,
"at the publication instant the new authority must already be live; \
observing the OLD one here means the authority swap escaped the epoch \
that names it"
);
assert!(
!Arc::ptr_eq(&a, &node.node_authority().expect("B is installed")),
"the transition under test must actually have replaced the authority"
);
}
#[tokio::test]
async fn an_authority_install_advances_the_routing_epoch_exactly_once() {
let node = node().await;
let org = crate::adapter::net::behavior::org::OrgKeypair::from_bytes([0x2du8; 32]);
node.install_node_authority(adopt_authority(&node, &org, "c"))
.expect("install A");
let before = node.routing_authority.epoch();
let advances: Arc<std::sync::atomic::AtomicU64> =
Arc::new(std::sync::atomic::AtomicU64::new(0));
{
let counter = advances.clone();
*node.routing_authority.post_publish_hook.lock() = Some(Arc::new(move |_| {
counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
}));
}
node.install_node_authority(adopt_authority(&node, &org, "d"))
.expect("install B");
*node.routing_authority.post_publish_hook.lock() = None;
assert_eq!(
advances.load(std::sync::atomic::Ordering::Acquire),
1,
"an authority+store install is ONE authority movement; a second advance \
means the two halves were published as separate ordered units"
);
let after = node.routing_authority.epoch();
assert!(
after > before,
"the routing epoch must move: {before} -> {after}"
);
}
#[tokio::test]
async fn a_refused_authority_install_publishes_neither_half() {
let node = node().await;
let org = crate::adapter::net::behavior::org::OrgKeypair::from_bytes([0x2eu8; 32]);
node.install_node_authority(adopt_authority(&node, &org, "e"))
.expect("install A");
let installed = node.node_authority().expect("A is installed");
let before = node.routing_authority.epoch();
let foreign = crate::adapter::net::behavior::org::OrgKeypair::from_bytes([0x9fu8; 32]);
node.install_node_authority(adopt_authority(&node, &foreign, "f"))
.expect_err("a foreign owner org must be refused at the one-owner preflight");
assert!(
Arc::ptr_eq(
&installed,
&node.node_authority().expect("A is still installed")
),
"a refused install must not publish its authority"
);
assert_eq!(
node.routing_authority.epoch(),
before,
"a refused install must not advance the routing epoch"
);
}
#[tokio::test]
async fn an_authority_rotation_over_the_same_store_still_publishes_inside_the_epoch() {
let node = node().await;
let org = crate::adapter::net::behavior::org::OrgKeypair::from_bytes([0x2fu8; 32]);
node.install_node_authority(adopt_authority(&node, &org, "g"))
.expect("install A");
let installed_store = node
.org_revocation
.load_full()
.expect("A's store is installed");
let replacement = adopt_authority(&node, &org, "h");
let advances = Arc::new(std::sync::atomic::AtomicU64::new(0));
let visible_in_callback = Arc::new(AtomicBool::new(false));
let early = Arc::new(AtomicBool::new(false));
{
let early = early.clone();
let premature = replacement.clone();
let weak = Arc::downgrade(&node);
*node.routing_authority.pre_publish_hook.lock() = Some(Arc::new(move |_epoch| {
if let Some(node) = weak.upgrade() {
if node
.node_authority()
.is_some_and(|live| Arc::ptr_eq(&live, &premature))
{
early.store(true, Ordering::Release);
}
}
}));
}
{
let advances = advances.clone();
let visible = visible_in_callback.clone();
let expected = replacement.clone();
let weak = Arc::downgrade(&node);
*node.routing_authority.post_publish_hook.lock() = Some(Arc::new(move |_epoch| {
advances.fetch_add(1, Ordering::AcqRel);
if let Some(node) = weak.upgrade() {
if node
.node_authority()
.is_some_and(|live| Arc::ptr_eq(&live, &expected))
{
visible.store(true, Ordering::Release);
}
}
}));
}
let before = node.routing_authority.epoch();
let publish = {
let replacement = replacement.clone();
let slot = &node.node_authority;
move || slot.store(Some(replacement.clone()))
};
let store_changed = {
let _install = node.org_install.lock();
node.install_org_revocation_store_locked(
installed_store.clone(),
false,
None,
Some(&publish as &(dyn Fn() + Sync)),
)
.expect("re-installing the exact same store is accepted")
};
*node.routing_authority.post_publish_hook.lock() = None;
*node.routing_authority.pre_publish_hook.lock() = None;
assert!(
!early.load(Ordering::Acquire),
"the replacement authority was already visible BEFORE the epoch advance — a reader in that window observes it under the OLD epoch identity"
);
assert!(
!store_changed,
"precondition: the same store `Arc` is not a visible store change — \
without this the test would silently be exercising the OTHER branch"
);
assert!(
Arc::ptr_eq(
&installed_store,
&node
.org_revocation
.load_full()
.expect("store still installed")
),
"precondition marker: the store is pointer-identical on this branch"
);
assert_eq!(
advances.load(Ordering::Acquire),
1,
"an authority-only rotation must still be ONE routing epoch transaction: \
0 means the authority was published outside the epoch entirely"
);
assert!(
visible_in_callback.load(Ordering::Acquire),
"at the publication instant the replacement authority must already be \
live; observing the old one means the swap escaped the epoch naming it"
);
assert!(
node.routing_authority.epoch() > before,
"the routing epoch must advance even though no store changed"
);
assert!(
Arc::ptr_eq(&replacement, &node.node_authority().expect("authority")),
"the rotation must actually have landed"
);
}
#[tokio::test]
async fn a_handles_lockfree_read_observes_the_registrys_published_artifact() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let node = node().await;
let scratch = Scratch::new("lockfree-cell", &node);
node.install_org_revocation_store(scratch.store())
.expect("install");
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: Arc::new(std::sync::atomic::AtomicU64::new(0)),
};
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(Arc::new(source), work, Arc::default());
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let key = slot(31, "nrpc:lockfree");
let held = family.demand(key.clone()).expect("demand");
assert!(
held.base_facts_unvalidated().is_none(),
"cold before the actor has installed anything"
);
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the install pass must settle: {outcome:?}"
);
let via_registry = registry
.base_facts_unvalidated(&key)
.expect("the actor installed an artifact");
let via_handle = held
.base_facts_unvalidated()
.expect("the handle must observe the actor's install; `None` here means the handle holds a cell the production install no longer writes to");
assert!(
Arc::ptr_eq(&via_handle, &via_registry),
"the lock-free and locked seams must return the SAME artifact — divergence means the install replaced the cell instead of storing into it, leaving every live handle permanently and silently cold"
);
registry.invalidate_if_stale(&key, &via_registry);
assert!(
held.base_facts_unvalidated().is_none(),
"an invalidation must be visible through the handle, or the warmed path would keep serving retired facts"
);
assert!(
registry.base_facts_unvalidated(&key).is_none(),
"and both seams must agree about the invalidation too"
);
}
async fn consumer_with_installed_grant(tag: &str) -> (Arc<MeshNode>, [u8; 32], [u8; 32]) {
use crate::adapter::net::behavior::org::OrgKeypair;
use crate::adapter::net::behavior::org_grant::{
GrantRights, GrantTargetScope, OrgCapabilityGrant,
};
let node = node().await;
let org = OrgKeypair::from_bytes([0xa1u8; 32]);
let issuer = OrgKeypair::from_bytes([0xa2u8; 32]);
node.install_node_authority(adopt_authority(&node, &org, tag))
.expect("install authority");
let provider = EntityKeypair::generate();
let (grant, secret) = OrgCapabilityGrant::try_issue(
&issuer,
org.org_id(),
CapabilityAuthorityId::for_tag("nrpc:pair-key"),
GrantRights::DISCOVER,
GrantTargetScope::ExactNode(provider.entity_id().clone()),
3600,
)
.expect("issue grant");
let secret = secret.expect("a DISCOVER grant mints a secret");
let grant_id = grant.grant_id;
node.install_consumer_grant_audience(grant, secret)
.expect("install consumer grant");
let handle = *node
.consumer_grant_audiences
.load()
.get(&grant_id)
.expect("the grant is installed")
.audience_handle();
(node, grant_id, handle)
}
#[tokio::test]
async fn a_stale_audience_handle_is_unserved_beside_its_installed_sibling() {
let (node, grant_id, installed_handle) = consumer_with_installed_grant("pairkey").await;
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let grant_slot = |audience_handle: [u8; 32]| SlotKey {
scope: PrivateAudienceScope::new(CapabilityAudienceScope::Grant {
grant_id,
audience_handle,
})
.expect("grant scopes are private"),
capability: CapabilityAuthorityId::for_tag("nrpc:pair-key"),
};
let installed_key = grant_slot(installed_handle);
let mut stale_handle = installed_handle;
stale_handle[0] ^= 0xff;
let stale_key = grant_slot(stale_handle);
assert_ne!(
installed_handle, stale_handle,
"precondition: the two scopes must differ in the handle ONLY"
);
let snapshot = source.snapshot(&[installed_key.clone(), stale_key.clone()]);
assert!(
matches!(
snapshot.providers(&installed_key).facts,
SourceFacts::Served(_)
),
"precondition: the scope whose handle IS installed must be served, or \
this witness proves nothing about the sibling"
);
assert!(
matches!(snapshot.providers(&stale_key).facts, SourceFacts::Unserved),
"a scope whose audience handle is not the installed one has NO evidence \
— serving it here hands the caller rows the installed handle authorizes \
under a scope the node has rotated away from"
);
let reversed = source.snapshot(&[stale_key.clone(), installed_key.clone()]);
assert!(
matches!(reversed.providers(&stale_key).facts, SourceFacts::Unserved),
"and the outcome must not depend on which sibling the batch reaches first"
);
assert!(
matches!(
reversed.providers(&installed_key).facts,
SourceFacts::Served(_)
),
"the installed scope stays served under either ordering"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_consumer_grant_removal_cannot_occupy_the_gap_between_validation_and_settlement() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let (node, grant_id, installed_handle) = consumer_with_installed_grant("wg7").await;
assert!(
node.org_revocation.load().is_some(),
"precondition: a store must be installed or the settlement gap is unreachable"
);
let source = ScopedSlotSource {
session_routing: node.session_routing.clone(),
scoped_discovery: node.scoped_discovery.clone(),
publication: node.scoped_publication.clone(),
org_revocation: node.org_revocation.clone(),
consumer_grants: node.consumer_grant_audiences.clone(),
consumer_grant_gate: node.consumer_grant_gate.clone(),
authority: node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: node.routing_unserved_scope.clone(),
};
let key = SlotKey {
scope: PrivateAudienceScope::new(CapabilityAudienceScope::Grant {
grant_id,
audience_handle: installed_handle,
})
.expect("grant scopes are private"),
capability: CapabilityAuthorityId::for_tag("nrpc:pair-key"),
};
let removed = Arc::new(AtomicBool::new(false));
let entered = Arc::new(AtomicBool::new(false));
let remover: Arc<parking_lot::Mutex<Option<std::thread::JoinHandle<()>>>> = Arc::default();
let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel::<()>(1);
let reached = Arc::new(parking_lot::Mutex::new(reached_rx));
let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
node.consumer_grant_gate
.arm_contended_hook(Arc::new(move || {
let _ = reached_tx.try_send(());
}));
{
let node = node.clone();
let removed = removed.clone();
let entered = entered.clone();
let remover = remover.clone();
let reached = reached.clone();
*source.settle_gap_hook.lock() = Some(Arc::new(move || {
entered.store(true, Ordering::Release);
let mover = node.clone();
let landed = removed.clone();
let done = done_tx.clone();
*remover.lock() = Some(std::thread::spawn(move || {
mover.remove_consumer_grant_audience(&grant_id);
landed.store(true, Ordering::Release);
let _ = done.send(());
}));
reached.lock().recv_timeout(Duration::from_secs(10)).expect(
"the remover's try_lock must FAIL, proving the commit pin holds \
the consumer-Grant gate; no signal means Grant movement is free \
to land between the validation and the settlement",
);
assert!(
!removed.load(Ordering::Acquire),
"a consumer-Grant removal landed between the validation and the \
settlement — the installation the pin validated is already \
withdrawn by the time the facts stamped with it are installed"
);
assert!(
node.consumer_grant_audiences
.load()
.get(&grant_id)
.is_some(),
"and the snapshot the settlement stamps against must still name \
the installation the pin validated"
);
}));
}
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(Arc::new(source), work, Arc::default());
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the install pass must settle: {outcome:?}"
);
assert!(
entered.load(Ordering::Acquire),
"the settlement gap must actually have been entered, or this witness \
asserted nothing"
);
assert!(
registry.base_facts_unvalidated(&key).is_some(),
"precondition: the quantum installed an artifact under the live grant"
);
done_rx
.recv_timeout(Duration::from_secs(10))
.expect("the removal must proceed once the pin releases the gate");
if let Some(handle) = remover.lock().take() {
handle.join().expect("remover thread");
}
assert!(
node.consumer_grant_audiences
.load()
.get(&grant_id)
.is_none(),
"the removal must actually have removed the grant"
);
drop(held);
}
struct GrantFixture {
node: Arc<MeshNode>,
org: crate::adapter::net::behavior::org::OrgKeypair,
issuer: crate::adapter::net::behavior::org::OrgKeypair,
provider: EntityKeypair,
}
async fn grant_fixture(tag: &str) -> GrantFixture {
use crate::adapter::net::behavior::org::{OrgKeypair, OrgMembershipCert};
use crate::adapter::net::behavior::org_authority::NodeAuthority;
use crate::adapter::net::identity::MAX_TOKEN_CLOCK_SKEW_SECS;
let node = node().await;
let org = OrgKeypair::from_bytes([0xa1u8; 32]);
let issuer = OrgKeypair::from_bytes([0xa2u8; 32]);
let entity = node.entity_id().clone();
let cert = OrgMembershipCert::try_issue(&org, entity.clone(), 1, 3600).expect("issue cert");
static SEQ: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let dir = std::env::temp_dir().join(format!("net-grantfx-{tag}-{}-{seq}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let authority = NodeAuthority::adopt(&dir, cert, &entity, MAX_TOKEN_CLOCK_SKEW_SECS, None)
.expect("adopt authority");
node.install_node_authority(Arc::new(authority))
.expect("install authority");
GrantFixture {
node,
org,
issuer,
provider: EntityKeypair::generate(),
}
}
fn copy_secret(
s: &crate::adapter::net::behavior::org_grant::OrgAudienceSecret,
) -> crate::adapter::net::behavior::org_grant::OrgAudienceSecret {
use crate::adapter::net::behavior::org_grant::OrgAudienceSecret;
let mut buf = s.encode_config();
let copy = OrgAudienceSecret::decode_config(&buf).expect("copy secret");
for b in buf.iter_mut() {
unsafe { std::ptr::write_volatile(b, 0) };
}
copy
}
impl GrantFixture {
fn mint(
&self,
tag: &str,
grant_id: Option<[u8; 32]>,
bounds: Option<(u64, u64)>,
) -> (
crate::adapter::net::behavior::org_grant::OrgCapabilityGrant,
crate::adapter::net::behavior::org_grant::OrgAudienceSecret,
) {
use crate::adapter::net::behavior::org::current_timestamp;
use crate::adapter::net::behavior::org_grant::{
GrantRights, GrantTargetScope, OrgAudienceSecret, OrgCapabilityGrant,
};
let target = GrantTargetScope::ExactNode(self.provider.entity_id().clone());
let cap = CapabilityAuthorityId::for_tag(tag);
let Some(grant_id) = grant_id else {
let (grant, secret) = OrgCapabilityGrant::try_issue(
&self.issuer,
self.org.org_id(),
cap,
GrantRights::DISCOVER,
target,
bounds.map_or(3600, |(_, exp)| exp.saturating_sub(current_timestamp())),
)
.expect("issue grant");
return (grant, secret.expect("a DISCOVER grant mints a secret"));
};
let now = current_timestamp();
let (secret, binding) = OrgAudienceSecret::mint(grant_id);
let (not_before, not_after) = bounds.unwrap_or((now.saturating_sub(60), now + 3600));
let grant = OrgCapabilityGrant::issue_at(
&self.issuer,
grant_id,
self.org.org_id(),
cap,
GrantRights::DISCOVER,
target,
Some(binding),
not_before,
not_after,
u64::from(grant_id[0]) ^ not_after,
);
(grant, secret)
}
fn install(
&self,
grant: crate::adapter::net::behavior::org_grant::OrgCapabilityGrant,
secret: crate::adapter::net::behavior::org_grant::OrgAudienceSecret,
) -> ([u8; 32], [u8; 32]) {
let grant_id = grant.grant_id;
self.node
.install_consumer_grant_audience(grant, secret)
.expect("install consumer grant");
let handle = *self
.node
.consumer_grant_audiences
.load()
.get(&grant_id)
.expect("the grant is installed")
.audience_handle();
(grant_id, handle)
}
fn key(&self, grant_id: [u8; 32], audience_handle: [u8; 32], tag: &str) -> SlotKey {
SlotKey {
scope: PrivateAudienceScope::new(CapabilityAudienceScope::Grant {
grant_id,
audience_handle,
})
.expect("grant scopes are private"),
capability: CapabilityAuthorityId::for_tag(tag),
}
}
fn warm(
&self,
key: &SlotKey,
) -> crate::adapter::net::behavior::org_routing_registry::DemandHandle {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RoutingHealth,
};
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
self.node
.routing_health
.store(Arc::new(RoutingHealth::Healthy { incarnation: 1 }));
self.node.routing_registry.activate_incarnation(1);
let family = self.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
let outcome = self.node.routing_registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the warming pass must settle, or nothing below is warm: {outcome:?}"
);
assert!(
self.node.org_routing_base_facts(key).is_some(),
"precondition: the slot must be WARM before the transition under test"
);
held
}
fn retain(
&self,
key: &SlotKey,
) -> crate::adapter::net::behavior::org_routing_registry::DemandHandle {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RoutingHealth,
};
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
self.node
.routing_health
.store(Arc::new(RoutingHealth::Healthy { incarnation: 1 }));
self.node.routing_registry.activate_incarnation(1);
let family = self.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
let outcome = self.node.routing_registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
matches!(outcome, ApplyOutcome::Current { .. }),
"the retaining pass must settle: {outcome:?}"
);
assert!(
self.node
.routing_registry
.base_facts_unvalidated(key)
.is_some(),
"precondition: the slot must hold an ARTIFACT before the transition"
);
held
}
fn source(&self) -> ScopedSlotSource {
ScopedSlotSource {
session_routing: self.node.session_routing.clone(),
scoped_discovery: self.node.scoped_discovery.clone(),
publication: self.node.scoped_publication.clone(),
org_revocation: self.node.org_revocation.clone(),
consumer_grants: self.node.consumer_grant_audiences.clone(),
consumer_grant_gate: self.node.consumer_grant_gate.clone(),
authority: self.node.routing_authority.clone(),
settle_gap_hook: parking_lot::Mutex::new(None),
unserved_scope: self.node.routing_unserved_scope.clone(),
}
}
}
#[tokio::test]
async fn a_same_id_grant_replacement_cannot_reauthorize_captured_facts() {
let f = grant_fixture("wg3").await;
let (grant, secret) = f.mint("nrpc:wg3", None, None);
let retained_grant = grant.clone();
let retained_secret = copy_secret(&secret);
let signature = grant.signature;
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:wg3");
let _held = f.warm(&key);
let warm = f.node.org_routing_base_facts(&key).expect("warm");
let ScopedDiscoveryAuthorityStamp::Grant {
grant_id: stamped_id,
install_seq: first_install_seq,
..
} = warm.authority
else {
panic!("precondition: the artifact must carry a GRANT stamp");
};
assert_eq!(
stamped_id, grant_id,
"precondition: the stamp names the installed grant"
);
assert!(
f.node.remove_consumer_grant_audience(&grant_id),
"precondition: the grant was installed"
);
let (reinstalled_id, reinstalled_handle) = f.install(retained_grant, retained_secret);
let installed = f.node.consumer_grant_audiences.load();
let record = installed.get(&grant_id).expect("reinstalled");
assert_eq!(reinstalled_id, grant_id, "case 1: the SAME grant id");
assert_eq!(
record.grant().signature,
signature,
"case 1: the SAME signed authority — byte-identical, not a re-issue"
);
assert_eq!(
reinstalled_handle, handle,
"case 1: the SAME audience handle"
);
assert_ne!(
record.install_seq(),
first_install_seq,
"case 1: and a DIFFERENT installation identity — the only thing that \
differs, and therefore the only thing that can retire the artifact"
);
drop(installed);
assert!(
f.node.org_routing_base_facts(&key).is_none(),
"facts captured under the PREVIOUS installation must not survive a \
remove-then-reinstall of the byte-identical grant: the signature and \
the handle are unchanged, so ONLY the non-aliasing installation \
identity can tell the two installations apart"
);
let _held2 = f.warm(&key);
assert!(
f.node.org_routing_base_facts(&key).is_some(),
"precondition: re-warmed under the reinstalled grant"
);
let (distinct, distinct_secret) = f.mint("nrpc:wg3-other", Some(grant_id), None);
assert_eq!(distinct.grant_id, grant_id, "case 2: the SAME grant id");
assert_ne!(
distinct.signature, signature,
"case 2: a DIFFERENT signed authority under that id"
);
assert!(f.node.remove_consumer_grant_audience(&grant_id));
f.install(distinct, distinct_secret);
assert!(
f.node.org_routing_base_facts(&key).is_none(),
"a DISTINCT signed grant reusing the id must not reauthorize facts \
captured under the grant it replaced"
);
}
#[tokio::test]
async fn the_signed_grant_identity_is_part_of_scope_currentness() {
use crate::adapter::net::behavior::org::current_timestamp;
let f = grant_fixture("wg4").await;
let (grant, secret) = f.mint("nrpc:wg4", None, None);
let (grant_id, handle) = f.install(grant, secret);
let installed = f.node.consumer_grant_audiences.load();
let record = installed.get(&grant_id).expect("installed");
let live = ScopedDiscoveryAuthorityStamp::Grant {
grant_id,
install_seq: record.install_seq(),
grant_signature: record.grant().signature,
audience_handle: handle,
};
let now = current_timestamp();
assert!(
f.node.scope_authority_is_current(&live, now),
"precondition: the exact installed authority is current"
);
let ScopedDiscoveryAuthorityStamp::Grant {
grant_signature, ..
} = live
else {
panic!("constructed as a Grant stamp");
};
let mut forged = grant_signature;
forged[0] ^= 0xff;
let tampered = ScopedDiscoveryAuthorityStamp::Grant {
grant_id,
install_seq: record.install_seq(),
grant_signature: forged,
audience_handle: handle,
};
assert!(
!f.node.scope_authority_is_current(&tampered, now),
"a stamp whose signed authority differs from the installed one is NOT \
current, even with the installation identity and handle equal — the \
signature binds the whole canonical grant, and dropping it from the \
comparison is invisible to every reachable-path witness"
);
}
#[tokio::test]
async fn the_audience_handle_is_part_of_scope_currentness() {
use crate::adapter::net::behavior::org::current_timestamp;
let f = grant_fixture("wg5").await;
let (grant, secret) = f.mint("nrpc:wg5", None, None);
let (grant_id, handle) = f.install(grant, secret);
let mut other = handle;
other[0] ^= 0xff;
let source = f.source();
let stale_key = f.key(grant_id, other, "nrpc:wg5");
let snapshot = source.snapshot(std::slice::from_ref(&stale_key));
assert!(
matches!(snapshot.providers(&stale_key).facts, SourceFacts::Unserved),
"a lone Grant key whose audience handle is not the installed one has NO \
evidence — this is the single-key case, with nothing else in the batch"
);
drop(snapshot);
let installed_key = f.key(grant_id, handle, "nrpc:wg5");
let ok = source.snapshot(std::slice::from_ref(&installed_key));
assert!(
matches!(ok.providers(&installed_key).facts, SourceFacts::Served(_)),
"precondition: the same key with the INSTALLED handle is served, so the \
refusal above is about the handle and not about the fixture"
);
drop(ok);
let installed = f.node.consumer_grant_audiences.load();
let record = installed.get(&grant_id).expect("installed");
let live = ScopedDiscoveryAuthorityStamp::Grant {
grant_id,
install_seq: record.install_seq(),
grant_signature: record.grant().signature,
audience_handle: handle,
};
let wrong_handle = ScopedDiscoveryAuthorityStamp::Grant {
grant_id,
install_seq: record.install_seq(),
grant_signature: record.grant().signature,
audience_handle: other,
};
let now = current_timestamp();
assert!(
f.node.scope_authority_is_current(&live, now),
"precondition: the exact installed authority is current"
);
assert!(
!f.node.scope_authority_is_current(&wrong_handle, now),
"a stamp whose audience handle differs from the installed record's is \
NOT current — the cached path must never compare less than the live \
query, which checks the handle as defence in depth"
);
}
#[tokio::test]
async fn a_grant_install_between_capture_and_commit_refuses_publication() {
use crate::adapter::net::behavior::org_routing::{
ApplyOutcome, ApplyRequest, DirtyApply, RegistryWork, RoutingHealth,
};
use crate::adapter::net::behavior::org_routing_registry::NodeOrgRoutingRegistry;
use crate::adapter::net::behavior::org_scoped_store::{
DirtyCapabilities, PrivateDiscoveryChangeBatch,
};
let f = grant_fixture("wg6").await;
let (grant_a, secret_a) = f.mint("nrpc:wg6", None, None);
let (id_a, handle_a) = f.install(grant_a, secret_a);
let key_a = f.key(id_a, handle_a, "nrpc:wg6");
let (grant_b, secret_b) = f.mint("nrpc:wg6-b", None, None);
let handle_b = secret_b.audience_handle;
let id_b = grant_b.grant_id;
let key_b = f.key(id_b, handle_b, "nrpc:wg6-b");
let during_build: Arc<parking_lot::Mutex<Option<BuildHook>>> = Arc::default();
let source = PausingSource {
inner: f.source(),
during_build: during_build.clone(),
after_pin: Arc::default(),
before_pin: Arc::default(),
};
let landed = Arc::new(AtomicBool::new(false));
{
let node = f.node.clone();
let landed = landed.clone();
let pending = parking_lot::Mutex::new(Some((grant_b, secret_b)));
*during_build.lock() = Some(Box::new(move || {
let Some((grant, secret)) = pending.lock().take() else {
return;
};
node.install_consumer_grant_audience(grant, secret)
.expect("the mid-capture install must itself succeed");
landed.store(true, Ordering::Release);
}));
}
let work: Arc<RegistryWork> = Arc::default();
let registry = NodeOrgRoutingRegistry::new(Arc::new(source), work, Arc::default());
f.node
.routing_health
.store(Arc::new(RoutingHealth::Healthy { incarnation: 1 }));
registry.activate_incarnation(1);
let family = registry.new_family().expect("family");
let held_a = family.demand(key_a.clone()).expect("demand a");
let held_b = family.demand(key_b.clone()).expect("demand b");
let outcome = registry.apply(
1,
ApplyRequest {
batch: PrivateDiscoveryChangeBatch {
generation: 0,
dirty: DirtyCapabilities::Clean,
},
registry_work: true,
},
);
assert!(
landed.load(Ordering::Acquire),
"the mid-capture install must actually have run, or this witness \
asserted nothing"
);
assert!(
matches!(outcome, ApplyOutcome::Superseded),
"an installation between capture and commit must defeat the pin: \
{outcome:?}"
);
assert!(
registry.base_facts_unvalidated(&key_a).is_none(),
"and NOTHING may be published — not even the key whose own Grant did \
not move, because the batch it was captured with is no longer current"
);
assert!(
registry.base_facts_unvalidated(&key_b).is_none(),
"least of all the key whose Grant arrived mid-capture"
);
drop(held_a);
drop(held_b);
}
#[tokio::test]
async fn unrelated_grant_movement_preserves_the_exact_slot() {
let f = grant_fixture("wg8").await;
let (grant_a, secret_a) = f.mint("nrpc:wg8-a", None, None);
let (id_a, handle_a) = f.install(grant_a, secret_a);
let key_a = f.key(id_a, handle_a, "nrpc:wg8-a");
let _held = f.warm(&key_a);
let warm = f
.node
.org_routing_base_facts(&key_a)
.expect("precondition: A's slot is warm");
let (grant_b, secret_b) = f.mint("nrpc:wg8-b", None, None);
let (id_b, _handle_b) = f.install(grant_b, secret_b);
assert_ne!(id_a, id_b, "precondition: the two grants are unrelated");
let after_install = f
.node
.org_routing_base_facts(&key_a)
.expect("installing an unrelated Grant must not cold A's slot");
assert!(
Arc::ptr_eq(&warm, &after_install),
"and must not merely leave it readable — it must be the EXACT same \
artifact, not a silently rebuilt one"
);
assert!(
f.node.remove_consumer_grant_audience(&id_b),
"precondition: B was installed"
);
let after_removal = f
.node
.org_routing_base_facts(&key_a)
.expect("removing an unrelated Grant must not cold A's slot either");
assert!(
Arc::ptr_eq(&warm, &after_removal),
"still the exact same artifact"
);
assert!(f.node.remove_consumer_grant_audience(&id_a));
assert!(
f.node.org_routing_base_facts(&key_a).is_none(),
"control: moving the slot's OWN Grant must cold it — without this the \
witness above is satisfied by a comparison that never fails"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn an_installed_grants_expiry_colds_its_facts_with_zero_providers() {
use crate::adapter::net::behavior::org::current_timestamp;
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
use crate::adapter::net::identity::MAX_TOKEN_CLOCK_SKEW_SECS;
let f = grant_fixture("wg13").await;
let base = current_timestamp();
const LEAD: u64 = 4;
let not_after = base + LEAD - MAX_TOKEN_CLOCK_SKEW_SECS;
let effective_deadline = not_after + MAX_TOKEN_CLOCK_SKEW_SECS;
assert_eq!(
effective_deadline,
base + LEAD,
"precondition: the arm is seconds away, not five minutes"
);
let (grant, secret) = f.mint(
"nrpc:wg13",
Some([0x13u8; 32]),
Some((base.saturating_sub(3600), not_after)),
);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:wg13");
let probe_key = f.key(grant_id, handle, "nrpc:wg13-probe");
f.node.start();
assert!(
until(|| f.node.org_routing_ready()).await,
"precondition: the actor must reach Healthy before anything is demanded"
);
let family = f.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
let held_probe = family.demand(probe_key.clone()).expect("demand probe");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()
&& f.node.org_routing_base_facts(&probe_key).is_some())
.await,
"precondition: the actor must warm BOTH slots under a currently-valid Grant"
);
let warm = f.node.org_routing_base_facts(&key).expect("warm");
assert!(
matches!(&warm.providers, SourceFacts::Served(p) if p.is_empty()),
"an installed current Grant with no providers is SERVED with exact empty \
evidence, not Unserved"
);
assert_eq!(
warm.earliest_expiry, effective_deadline,
"THE point of this witness: with ZERO provider rows the artifact's only \
possible deadline is its AUTHORITY's. Derived from rows alone this is \
u64::MAX — an artifact claiming never to expire while the Grant behind \
it expires in seconds, with nothing armed to notice"
);
assert_eq!(
f.node.routing_registry.next_artifact_deadline(),
Some(effective_deadline),
"and the registry must ARM to it — this is what the actor sleeps on, so \
a deadline that never reaches here wakes nobody"
);
assert!(
f.node
.org_routing_base_facts_at(&probe_key, effective_deadline.saturating_sub(1))
.is_some(),
"precondition: still current one second before the effective deadline"
);
assert!(
f.node
.org_routing_base_facts_at(&probe_key, effective_deadline)
.is_none(),
"an installed-but-expired Grant authorizes NOTHING, and the boundary is exact: `now >= not_after + skew`, matching `check_time_bounds_at`"
);
assert_eq!(
f.node.routing_registry.next_artifact_deadline(),
Some(effective_deadline),
"and the actor's OWN slot is still armed after that probe — proof the read-seam half is not what retires it below"
);
assert!(
until(|| f.node.routing_registry.next_artifact_deadline().is_none()).await,
"the actor must wake on its OWN deadline arm and retire the expired \
artifact; a deadline still armed here means nothing consumed it"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"and the requeued rebuild must settle UNSERVED: past its deadline the \
installed Grant authorizes nothing, so the scope has no evidence at all"
);
assert!(
f.node.org_routing_base_facts(&key).is_none(),
"the read seam is cold — and now AGREES with the retained set rather \
than being the only thing that knew"
);
let settled = f.node.org_routing_reconciliation_counts();
tokio::time::sleep(Duration::from_millis(300)).await;
assert_eq!(
f.node.org_routing_reconciliation_counts()[0],
settled[0],
"the actor must be QUIESCENT once the expired artifact is retired — a climbing install count means the deadline arm re-fires on what it just rebuilt"
);
assert!(
f.node.routing_registry.next_artifact_deadline().is_none(),
"and nothing is armed: `Unserved` carries no deadline, which is what makes the retirement terminal rather than cyclic"
);
drop(held);
drop(held_probe);
let _ = f.node.shutdown().await;
assert!(
f.node.routing_task.lock().is_none(),
"every spawned task must be joined"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn installing_a_consumer_grant_wakes_the_affected_grant_slot() {
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
let f = grant_fixture("ww1").await;
let (grant, secret) = f.mint("nrpc:ww1", Some([0xd1u8; 32]), None);
let grant_id = grant.grant_id;
let handle = secret.audience_handle;
let key = f.key(grant_id, handle, "nrpc:ww1");
f.node.start();
assert!(
until(|| f.node.org_routing_ready()).await,
"precondition: the actor must reach Healthy"
);
let family = f.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"precondition: with no installed Grant the slot reconstructs UNSERVED"
);
assert!(
f.node.org_routing_base_facts(&key).is_none(),
"precondition: and reads cold — note WITHOUT invalidating, which is why \
no reader can ever rescue this slot"
);
let scoped_before = f.node.scoped_discovery.lock().revision();
f.install(grant, secret);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Served(_)))
})
.await,
"installing the Grant must WAKE routing and re-serve the slot; without \
the install notification nothing moves and it stays Unserved forever"
);
assert!(
f.node.org_routing_base_facts(&key).is_some(),
"and the read seam must now serve it"
);
assert_eq!(
f.node.scoped_discovery.lock().revision(),
scoped_before,
"and the scoped revision must NOT have moved — item 11: a Grant \
transition does not mutate the scoped store, so the scoped revision \
cannot be what triggered this"
);
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn removing_a_consumer_grant_wakes_the_affected_grant_slot() {
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
let f = grant_fixture("ww2").await;
let (grant, secret) = f.mint("nrpc:ww2", None, None);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:ww2");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"precondition: the slot is warm under the installed Grant"
);
let scoped_before = f.node.scoped_discovery.lock().revision();
assert!(
f.node.remove_consumer_grant_audience(&grant_id),
"precondition: the grant was installed"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"removing the Grant must WAKE routing and rebuild the slot as Unserved, \
with no reader involved; without the removal notification the stale \
artifact stays retained until something happens to read it"
);
assert_eq!(
f.node.scoped_discovery.lock().revision(),
scoped_before,
"and again NOT via the scoped revision (item 11)"
);
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test]
async fn consumer_grant_movement_wakes_only_the_affected_scope() {
let f = grant_fixture("ww3").await;
let (grant_a, secret_a) = f.mint("nrpc:ww3-a", None, None);
let (id_a, handle_a) = f.install(grant_a, secret_a);
let key_a = f.key(id_a, handle_a, "nrpc:ww3-a");
let owner_key = slot(44, "nrpc:ww3-owner");
let _held_a = f.warm(&key_a);
let _held_owner = f.warm(&owner_key);
let warm_a = f.node.org_routing_base_facts(&key_a).expect("A warm");
let warm_owner = f
.node
.org_routing_base_facts(&owner_key)
.expect("owner warm");
let (grant_b, secret_b) = f.mint("nrpc:ww3-b", None, None);
let (id_b, _handle_b) = f.install(grant_b, secret_b);
assert_ne!(id_a, id_b, "precondition: unrelated grants");
assert!(
f.node
.org_routing_base_facts(&key_a)
.is_some_and(|live| Arc::ptr_eq(&live, &warm_a)),
"after B install: A's EXACT artifact must survive unrelated Grant \
movement — not merely be readable, but be the same artifact"
);
assert!(
f.node
.org_routing_base_facts(&owner_key)
.is_some_and(|live| Arc::ptr_eq(&live, &warm_owner)),
"after B install: and the Owner plane must be untouched — it has no \
consumer Grant to move"
);
assert!(f.node.remove_consumer_grant_audience(&id_b));
assert!(
f.node
.org_routing_base_facts(&key_a)
.is_some_and(|live| Arc::ptr_eq(&live, &warm_a)),
"after B removal: still A's exact artifact"
);
assert!(
f.node
.org_routing_base_facts(&owner_key)
.is_some_and(|live| Arc::ptr_eq(&live, &warm_owner)),
"after B removal: Owner still untouched"
);
assert!(f.node.remove_consumer_grant_audience(&id_a));
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key_a)
.is_none(),
"control: moving the slot's OWN Grant must retire its artifact"
);
assert!(
f.node
.org_routing_base_facts(&owner_key)
.is_some_and(|live| Arc::ptr_eq(&live, &warm_owner)),
"control: and even then the Owner plane is spared"
);
}
#[tokio::test]
async fn a_grant_movement_notification_runs_after_publication_and_after_release() {
let f = grant_fixture("ww4").await;
let observations: Arc<parking_lot::Mutex<Vec<(bool, bool)>>> = Arc::default();
{
let sink = observations.clone();
let weak = Arc::downgrade(&f.node);
f.node.arm_grant_movement_hook(Arc::new(move |movement| {
let Some(node) = weak.upgrade() else {
return;
};
let gate_released = node.consumer_grant_gate.mu.try_lock().is_some();
let published = node
.consumer_grant_audiences
.load()
.get(&movement.grant_id)
.is_some();
sink.lock().push((gate_released, published));
}));
}
let (grant, secret) = f.mint("nrpc:ww4", None, None);
let (grant_id, _handle) = f.install(grant, secret);
{
let seen = observations.lock().clone();
assert_eq!(seen.len(), 1, "one notification for one install");
assert!(
seen[0].0,
"the consumer-Grant gate must be RELEASED at the notification \
instant — holding it here is the gate -> registry inversion of the \
commit pin's own order"
);
assert!(
seen[0].1,
"and the new snapshot must ALREADY be published — a wake that \
precedes publication lets the actor rebuild against the old \
snapshot and reinstall the staleness it was woken to clear"
);
}
observations.lock().clear();
assert!(f.node.remove_consumer_grant_audience(&grant_id));
let seen = observations.lock().clone();
assert_eq!(seen.len(), 1, "one notification for one removal");
assert!(seen[0].0, "gate released on the removal path too");
assert!(
!seen[0].1,
"and the removal must ALREADY be published — the snapshot must no longer \
carry the grant when routing is told it moved"
);
}
#[tokio::test]
async fn a_non_publishing_grant_outcome_wakes_nothing() {
use crate::adapter::net::behavior::org_grant_registry::ConsumerAudienceLease;
let f = grant_fixture("ww5").await;
let (grant, secret) = f.mint("nrpc:ww5", None, None);
let retained = grant.clone();
let retained_secret = copy_secret(&secret);
let (grant_id, _handle) = f.install(grant, secret);
let after_install = f.node.consumer_grant_movements_for_test();
assert_eq!(after_install, 1, "precondition: the real install woke once");
f.node
.install_consumer_grant_audience(retained, retained_secret)
.expect("idempotent install is valid");
assert_eq!(
f.node.consumer_grant_movements_for_test(),
after_install,
"an IDEMPOTENT install publishes nothing and must wake nothing"
);
let stale = ConsumerAudienceLease::new(grant_id, u64::MAX);
assert!(
!f.node.remove_consumer_grant_audience_if_current(&stale),
"precondition: a stale lease owns nothing"
);
assert_eq!(
f.node.consumer_grant_movements_for_test(),
after_install,
"a STALE lease publishes nothing and must wake nothing — otherwise a \
superseded lease holder can churn the live installation's slots at will"
);
assert!(!f.node.remove_consumer_grant_audience(&[0xffu8; 32]));
assert_eq!(
f.node.consumer_grant_movements_for_test(),
after_install,
"a no-op removal publishes nothing and must wake nothing"
);
assert!(f.node.remove_consumer_grant_audience(&grant_id));
assert_eq!(
f.node.consumer_grant_movements_for_test(),
after_install + 1,
"control: a real publication DOES wake — without this the assertions \
above pass against a counter that never moves"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delayed_grant_notification_cannot_retire_a_successor_installation() {
let f = grant_fixture("ww6").await;
let (grant, secret) = f.mint("nrpc:ww6", Some([0xa6u8; 32]), None);
let retained_grant = grant.clone();
let retained_secret = copy_secret(&secret);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:ww6");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"precondition: warm under installation N"
);
let (reached_rx, release_tx, _parked_fence) = park_first_grant_notification(&f.node);
let remover = {
let node = f.node.clone();
std::thread::spawn(move || node.remove_consumer_grant_audience(&grant_id))
};
reached_rx
.recv_timeout(Duration::from_secs(10))
.expect("the removal must reach its notification and park there");
f.node
.install_consumer_grant_audience(retained_grant, retained_secret)
.expect("reinstall");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"the successor installation must warm the slot again"
);
let successor = f
.node
.org_routing_base_facts(&key)
.expect("successor artifact");
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release_tx.send(());
assert!(
remover.join().expect("remover thread"),
"precondition: the removal itself did publish"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &successor)),
"the delayed notification for the SUPERSEDED installation must leave the \
successor artifact exactly as it was — pointer identity, not merely \
eventual readability"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(
counts_after[2], counts_before[2],
"and must invalidate nothing: an obsolete transition retiring current \
work is the defect, even though the read seam would stay fail-closed"
);
assert_eq!(
counts_after[0], counts_before[0],
"and must not have re-queued it either — a rebuild here means the \
successor was needlessly cold-pathed by an obsolete transition"
);
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test]
async fn consumer_grant_movement_preserves_same_id_unaffected_scopes() {
let f = grant_fixture("ww7").await;
let (grant, secret) = f.mint("nrpc:ww7", Some([0xa7u8; 32]), None);
let (grant_id, handle) = f.install(grant, secret);
let moved = f.key(grant_id, handle, "nrpc:ww7");
let mut stale_handle = handle;
stale_handle[0] ^= 0xff;
let stale_scope = f.key(grant_id, stale_handle, "nrpc:ww7");
let other_capability = f.key(grant_id, handle, "nrpc:ww7-other-capability");
let (grant_b, secret_b) = f.mint("nrpc:ww7-b", None, None);
let (id_b, handle_b) = f.install(grant_b, secret_b);
let unrelated = f.key(id_b, handle_b, "nrpc:ww7-b");
let owner = slot(45, "nrpc:ww7-owner");
let _h1 = f.warm(&moved);
let _h2 = f.retain(&stale_scope);
let _h3 = f.warm(&other_capability);
let _h4 = f.warm(&unrelated);
let _h5 = f.warm(&owner);
let before_stale = f.node.routing_registry.base_facts_unvalidated(&stale_scope);
let before_unrelated = f
.node
.org_routing_base_facts(&unrelated)
.expect("unrelated warm");
let before_owner = f.node.org_routing_base_facts(&owner).expect("owner warm");
assert!(
f.node.org_routing_base_facts(&other_capability).is_some(),
"precondition: an uncovered capability under an installed Grant is \
SERVED (empty), which is why it is treated as affected below"
);
assert!(f.node.remove_consumer_grant_audience(&grant_id));
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&moved)
.is_none(),
"precondition: the scope that actually moved IS retired"
);
assert!(
f.node
.org_routing_base_facts(&unrelated)
.is_some_and(|live| Arc::ptr_eq(&live, &before_unrelated)),
"an unrelated Grant keeps its EXACT artifact"
);
assert!(
f.node
.org_routing_base_facts(&owner)
.is_some_and(|live| Arc::ptr_eq(&live, &before_owner)),
"and the Owner plane is untouched"
);
match before_stale {
Some(before) => assert!(
f.node
.routing_registry
.base_facts_unvalidated(&stale_scope)
.is_some_and(|live| Arc::ptr_eq(&live, &before)),
"the SAME grant id under a rotated-away audience handle is a \
different scope: this transition says nothing about it, so its \
exact artifact must survive"
),
None => panic!("precondition: the stale-handle slot must be retained"),
}
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&other_capability)
.is_none(),
"and the same scope under a different capability IS retired — see this \
witness's doc: the source serves it, so the grant's movement affects \
it. If the source is ever narrowed to refuse uncovered capabilities, \
THIS assertion is the one that must fail and force the invalidation to \
be narrowed with it"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delayed_install_notification_cannot_retire_a_successor_removal_artifact() {
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
let f = grant_fixture("ww8").await;
let (grant, secret) = f.mint("nrpc:ww8", Some([0xa8u8; 32]), None);
let grant_id = grant.grant_id;
let handle = secret.audience_handle;
let key = f.key(grant_id, handle, "nrpc:ww8");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let (reached_rx, release_tx, _parked_fence) = park_first_grant_notification(&f.node);
let installer = {
let node = f.node.clone();
std::thread::spawn(move || {
node.install_consumer_grant_audience(grant, secret)
.expect("install")
})
};
reached_rx
.recv_timeout(Duration::from_secs(10))
.expect("the install must reach its notification and park there");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"the slot must warm under the installed grant before it is removed"
);
assert!(
f.node.remove_consumer_grant_audience(&grant_id),
"precondition: the removal published"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"the removal must leave a newer UNSERVED artifact — retained, and read \
cold"
);
let successor = f
.node
.routing_registry
.base_facts_unvalidated(&key)
.expect("successor absence artifact");
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release_tx.send(());
installer.join().expect("installer thread");
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &successor)),
"the obsolete INSTALL notification must preserve the exact newer \
Unserved artifact produced by the later removal — an absence IS a \
successor, and ordering by installation identity cannot see it"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(
counts_after[2], counts_before[2],
"and must invalidate nothing"
);
assert_eq!(
counts_after[0], counts_before[0],
"and must not have re-queued it either"
);
drop(held);
let _ = f.node.shutdown().await;
}
fn park_first_grant_notification(
node: &Arc<MeshNode>,
) -> (
std::sync::mpsc::Receiver<()>,
std::sync::mpsc::SyncSender<()>,
Arc<parking_lot::Mutex<Option<GrantMovementFence>>>,
) {
let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel::<()>(1);
let (release_tx, release_rx) = std::sync::mpsc::sync_channel::<()>(1);
let release_rx = Arc::new(parking_lot::Mutex::new(release_rx));
let parked = Arc::new(AtomicBool::new(false));
let fence = Arc::new(parking_lot::Mutex::new(None));
let seen = fence.clone();
node.arm_grant_movement_hook(Arc::new(move |movement| {
if parked.swap(true, Ordering::AcqRel) {
return;
}
*seen.lock() = Some(movement.fence);
let _ = reached_tx.try_send(());
release_rx
.lock()
.recv_timeout(Duration::from_secs(10))
.expect("the witness must release the parked notification");
}));
(reached_rx, release_tx, fence)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delayed_install_notification_preserves_its_own_publication_artifact() {
let f = grant_fixture("ww9").await;
let (grant, secret) = f.mint("nrpc:ww9", Some([0xa9u8; 32]), None);
let grant_id = grant.grant_id;
let handle = secret.audience_handle;
let key = f.key(grant_id, handle, "nrpc:ww9");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let (reached, release, parked_fence) = park_first_grant_notification(&f.node);
let installer = {
let node = f.node.clone();
std::thread::spawn(move || {
node.install_consumer_grant_audience(grant, secret)
.expect("install")
})
};
reached
.recv_timeout(Duration::from_secs(10))
.expect("the install must park at its notification");
assert!(
matches!(
*parked_fence.lock(),
Some(GrantMovementFence::Publication(_))
),
"precondition: the parked transition is an ORDINARY publication — this \
witness is about the equality arm, not the terminal row"
);
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"the demand must warm the slot under the just-published grant"
);
let own = f.node.org_routing_base_facts(&key).expect("own artifact");
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release.send(());
installer.join().expect("installer thread");
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &own)),
"a notification must preserve the artifact its OWN publication produced \
— the comparison is STRICTLY less than, and `<=` would have every \
install that races a demand cold-path its own work"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(counts_after[2], counts_before[2], "and invalidate nothing");
assert_eq!(counts_after[0], counts_before[0], "and re-queue nothing");
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delayed_removal_notification_preserves_its_own_absence_artifact() {
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
let f = grant_fixture("ww10").await;
let (grant, secret) = f.mint("nrpc:ww10", Some([0xaau8; 32]), None);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:ww10");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let (reached, release, parked_fence) = park_first_grant_notification(&f.node);
let remover = {
let node = f.node.clone();
std::thread::spawn(move || node.remove_consumer_grant_audience(&grant_id))
};
reached
.recv_timeout(Duration::from_secs(10))
.expect("the removal must park at its notification");
assert!(
matches!(
*parked_fence.lock(),
Some(GrantMovementFence::Publication(_))
),
"precondition: the parked withdrawal is an ORDINARY publication. W-W15 \
is the same schedule at the LAST live identity, where the artifact \
side is terminal instead"
);
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"the demand must reconstruct the slot as Unserved under the published \
absence"
);
let own = f
.node
.routing_registry
.base_facts_unvalidated(&key)
.expect("own absence artifact");
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release.send(());
assert!(
remover.join().expect("remover thread"),
"precondition: the removal published"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &own)),
"a removal notification must preserve the ABSENCE artifact its own \
publication produced, exactly as an install preserves its Grant one — \
the ordering is over transitions, not over authority states"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(counts_after[2], counts_before[2], "and invalidate nothing");
assert_eq!(counts_after[0], counts_before[0], "and re-queue nothing");
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_current_lease_removal_publishes_wakes_and_fences() {
use crate::adapter::net::behavior::org_grant_registry::ConsumerAudienceInstall;
use crate::adapter::net::behavior::org_routing_registry::SourceFacts;
let f = grant_fixture("ww11").await;
let (grant, secret) = f.mint("nrpc:ww11", Some([0xabu8; 32]), None);
let grant_id = grant.grant_id;
let handle = secret.audience_handle;
let lease = match f
.node
.install_consumer_grant_audience_leased(grant, secret)
.expect("install")
{
ConsumerAudienceInstall::Installed(lease) => lease,
ConsumerAudienceInstall::AlreadyPresent => panic!("a fresh install must yield a lease"),
};
let key = f.key(grant_id, handle, "nrpc:ww11");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| f.node.org_routing_base_facts(&key).is_some()).await,
"precondition: warm under the leased installation"
);
let movements_before = f.node.consumer_grant_movements_for_test();
assert!(
f.node.remove_consumer_grant_audience_if_current(&lease),
"a current lease must withdraw its own installation"
);
assert_eq!(
f.node.consumer_grant_movements_for_test(),
movements_before + 1,
"and that withdrawal is routing movement — exactly one notification"
);
assert!(
f.node
.consumer_grant_audiences
.load()
.get(&grant_id)
.is_none(),
"and absence is published"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"the conditional removal must wake routing and rebuild the slot as \
Unserved, without a reader"
);
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test]
async fn publication_exhaustion_refuses_an_install_without_publishing() {
use crate::adapter::net::behavior::org_grant_registry::GrantAudienceInstallError;
let f = grant_fixture("ww12").await;
let (grant, secret) = f.mint("nrpc:ww12", Some([0xacu8; 32]), None);
f.node.exhaust_consumer_grant_publications_for_test();
let snapshot_before = f.node.consumer_grant_audiences.load_full();
let identity_before = f.node.consumer_grant_publication_for_test();
let movements_before = f.node.consumer_grant_movements_for_test();
let published_before = f.node.consumer_grant_publications_for_test();
let err = f
.node
.install_consumer_grant_audience(grant, secret)
.expect_err("an exhausted publication space must refuse an install");
assert_eq!(
err,
GrantAudienceInstallError::IdSpaceExhausted,
"the refusal must be TYPED. It shares the public variant with installation-identity exhaustion deliberately: a NEW variant on that public enum would be a source-breaking API change, which scope item 16 excludes. The precise space is carried in the log, and the counter assertions below are what distinguish the two behaviourally"
);
assert!(
Arc::ptr_eq(
&snapshot_before,
&f.node.consumer_grant_audiences.load_full()
),
"NO PARTIAL PUBLICATION: the exact snapshot from before must still be \
installed. The defect was that content became visible and only THEN \
failed"
);
assert_eq!(
f.node.consumer_grant_publications_for_test(),
published_before,
"and nothing was published even transiently"
);
assert_eq!(
f.node.consumer_grant_publication_for_test(),
identity_before,
"and the identity did not advance, wrap, or saturate"
);
assert_ne!(
f.node.consumer_grant_publication_for_test(),
0,
"least of all alias to zero, against which every retained artifact \
compares as newer and nothing is ever invalidated again"
);
assert_eq!(
f.node.consumer_grant_movements_for_test(),
movements_before,
"and a refusal is not routing movement"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn terminal_withdrawal_retires_the_last_live_identity_and_nothing_else() {
use crate::adapter::net::behavior::org_routing_registry::{GrantArtifactFence, SourceFacts};
let f = grant_fixture("ww13").await;
let (grant_b, secret_b) = f.mint("nrpc:ww13-b", Some([0xaeu8; 32]), None);
let (id_b, handle_b) = f.install(grant_b, secret_b);
let unrelated = f.key(id_b, handle_b, "nrpc:ww13-b");
let owner = slot(46, "nrpc:ww13-owner");
let (grant, secret) = f.mint("nrpc:ww13", Some([0xadu8; 32]), None);
let grant_id = grant.grant_id;
let handle = secret.audience_handle;
let key = f.key(grant_id, handle, "nrpc:ww13");
let key_alt = f.key(grant_id, handle, "nrpc:ww13-alt");
let mut other_handle = handle;
other_handle[0] ^= 0xff;
let same_id_other_handle = f.key(grant_id, other_handle, "nrpc:ww13");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let held_h = family
.demand(same_id_other_handle.clone())
.expect("demand h");
let held_b = family.demand(unrelated.clone()).expect("demand b");
let held_o = family.demand(owner.clone()).expect("demand owner");
assert!(
until(|| {
[&same_id_other_handle, &unrelated, &owner]
.iter()
.all(|k| f.node.routing_registry.base_facts_unvalidated(k).is_some())
})
.await,
"precondition: every control holds an artifact"
);
let before_h = f
.node
.routing_registry
.base_facts_unvalidated(&same_id_other_handle)
.expect("h");
assert!(
matches!(before_h.grant_fence, GrantArtifactFence::Publication(_)),
"precondition, and the one this witness was rebuilt for: the \
same-id/other-handle control must carry an ORDINARY publication. \
Stamped TerminalAbsence it would be preserved by the fence even when \
wrongly selected, and a grant_id-only terminal predicate would pass \
unnoticed"
);
f.node
.set_consumer_grant_publications_for_test(u64::MAX - 2);
let (grant_id_installed, handle_installed) = f.install(grant, secret);
assert_eq!(
(grant_id_installed, handle_installed),
(grant_id, handle),
"precondition: the installed record carries the scope the keys above \
were built from"
);
assert_eq!(
f.node.consumer_grant_publication_for_test(),
u64::MAX - 1,
"precondition: the installation commits the LAST live identity — the \
whole point of this witness is that the artifact carries exactly the \
value a broken terminal fence would reuse"
);
let held = family.demand(key.clone()).expect("demand");
let held_alt = family.demand(key_alt.clone()).expect("demand alt");
assert!(
until(|| {
[&key, &key_alt]
.iter()
.all(|k| f.node.routing_registry.base_facts_unvalidated(k).is_some())
})
.await,
"precondition: both capabilities under the exact scope hold artifacts"
);
for (label, k) in [
("the granted capability", &key),
("a second capability under the same exact scope", &key_alt),
] {
let warm = f
.node
.routing_registry
.base_facts_unvalidated(k)
.expect("warm");
assert_eq!(
warm.grant_fence,
GrantArtifactFence::Publication(u64::MAX - 1),
"{label}: must be stamped with the LAST LIVE identity, which is the \
value a reusing implementation would compare against itself"
);
}
let before_b = f
.node
.routing_registry
.base_facts_unvalidated(&unrelated)
.expect("b");
let before_o = f
.node
.routing_registry
.base_facts_unvalidated(&owner)
.expect("owner");
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&same_id_other_handle)
.is_some_and(|live| Arc::ptr_eq(&live, &before_h)),
"precondition: the INSTALL is exact too, so the control still holds the \
ordinary-publication artifact it warmed with"
);
let counts_before = quiesced_counts(&f.node).await;
assert!(
f.node.remove_consumer_grant_audience(&grant_id),
"an exhausted publication space must NOT block revocation"
);
assert_eq!(
f.node.consumer_grant_publication_for_test(),
u64::MAX - 1,
"and must not advance past the last live value — `u64::MAX` stays \
reserved as the terminal marker"
);
assert!(
until(|| {
[&key, &key_alt].iter().all(|k| {
f.node
.routing_registry
.base_facts_unvalidated(k)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
})
.await,
"the withdrawal must retire BOTH capability slots stamped MAX-1 and \
rebuild them as Unserved. An implementation that reused \
Publication(MAX-1) here would compare it against itself and leave the \
stale Served artifacts in place; one narrowed to the grant's own \
capability would leave the second slot Grant-stamped after removal"
);
for (label, key, before) in [
(
"same grant id, other audience handle",
&same_id_other_handle,
&before_h,
),
("an unrelated grant", &unrelated, &before_b),
("the Owner plane", &owner, &before_o),
] {
assert!(
f.node
.routing_registry
.base_facts_unvalidated(key)
.is_some_and(|live| Arc::ptr_eq(&live, before)),
"{label}: must keep its EXACT artifact. `Terminal` skips the \
generation comparison entirely, so it is the easiest fence to \
widen by accident"
);
}
let settled = f.node.org_routing_reconciliation_counts();
tokio::time::sleep(Duration::from_millis(300)).await;
let quiet = f.node.org_routing_reconciliation_counts();
assert_eq!(
quiet[0], settled[0],
"the actor must be QUIESCENT after a terminal fence"
);
assert_eq!(
quiet[2],
counts_before[2] + 2,
"EXACTLY the two capability slots under the exact scope were \
invalidated. `>=` was the earlier assertion and it is vacuous — it \
passes at zero, which is the retirement failure this witness exists to \
catch, and it passes at five, which is the scope widening"
);
assert_eq!(
quiet[0],
counts_before[0] + 2,
"and exactly those two were rebuilt — no control was re-queued"
);
drop(held);
drop(held_alt);
drop(held_h);
drop(held_b);
drop(held_o);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delayed_terminal_notification_preserves_its_own_absence_artifact() {
use crate::adapter::net::behavior::org_routing_registry::{GrantArtifactFence, SourceFacts};
let f = grant_fixture("ww14").await;
f.node
.set_consumer_grant_publications_for_test(u64::MAX - 2);
let (grant, secret) = f.mint("nrpc:ww14", Some([0xafu8; 32]), None);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:ww14");
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let (reached, release, parked_fence) = park_first_grant_notification(&f.node);
let remover = {
let node = f.node.clone();
std::thread::spawn(move || node.remove_consumer_grant_audience(&grant_id))
};
reached
.recv_timeout(Duration::from_secs(10))
.expect("the terminal withdrawal must park at its notification");
assert_eq!(
*parked_fence.lock(),
Some(GrantMovementFence::Terminal),
"precondition, asserted DIRECTLY rather than inferred from the setup: \
the parked movement is the TERMINAL row of the matrix. Deriving it \
from `set_consumer_grant_publications_for_test` plus an install would \
let a reservation change silently turn this into an ordinary \
publication, which W-W10 already covers"
);
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"the demand must reconstruct the scope as Unserved under the published \
absence"
);
let own = f
.node
.routing_registry
.base_facts_unvalidated(&key)
.expect("own terminal absence artifact");
assert_eq!(
own.grant_fence,
GrantArtifactFence::TerminalAbsence,
"precondition: an absent Grant scope reconstructed under a SPENT identity \
space is terminal — and it is a distinct fence, not a number, precisely \
because no number could order it against the withdrawal that caused it"
);
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release.send(());
assert!(
remover.join().expect("remover thread"),
"precondition: the withdrawal published"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &own)),
"a terminal withdrawal must preserve the absence artifact its OWN \
publication produced — clearing everything unconditionally retires \
current work that nothing owes"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(counts_after[2], counts_before[2], "and invalidate nothing");
assert_eq!(counts_after[0], counts_before[0], "and re-queue nothing");
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_final_ordinary_removal_preserves_the_terminal_absence_it_caused() {
use crate::adapter::net::behavior::org_routing_registry::{GrantArtifactFence, SourceFacts};
let f = grant_fixture("ww15").await;
let (grant, secret) = f.mint("nrpc:ww15", Some([0xb0u8; 32]), None);
let (grant_id, handle) = f.install(grant, secret);
let key = f.key(grant_id, handle, "nrpc:ww15");
f.node
.set_consumer_grant_publications_for_test(u64::MAX - 2);
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let (reached, release, parked_fence) = park_first_grant_notification(&f.node);
let remover = {
let node = f.node.clone();
std::thread::spawn(move || node.remove_consumer_grant_audience(&grant_id))
};
reached
.recv_timeout(Duration::from_secs(10))
.expect("the final ordinary removal must park at its notification");
assert_eq!(
*parked_fence.lock(),
Some(GrantMovementFence::Publication(u64::MAX - 1)),
"precondition, and the distinction this witness rests on: the parked \
movement is an ORDINARY publication at the last live identity, NOT \
Terminal. Withdrawal reserves before it publishes, and MAX-1 was still \
available"
);
assert_eq!(
f.node.consumer_grant_publication_for_test(),
u64::MAX - 1,
"and the space is now spent: the next reservation would land on the \
reserved terminal marker"
);
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"the demand must reconstruct the scope as Unserved under the published \
absence"
);
let own = f
.node
.routing_registry
.base_facts_unvalidated(&key)
.expect("own terminal absence artifact");
assert_eq!(
own.grant_fence,
GrantArtifactFence::TerminalAbsence,
"precondition: reconstructed absent under a SPENT space, so the artifact \
is terminal even though the movement that caused it was ordinary"
);
let counts_before = f.node.org_routing_reconciliation_counts();
let _ = release.send(());
assert!(
remover.join().expect("remover thread"),
"precondition: the removal published"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key)
.is_some_and(|live| Arc::ptr_eq(&live, &own)),
"an ordinary notification must preserve the TERMINAL absence its own \
publication produced. Nothing can supersede that artifact — no \
installation can follow a spent space — so clearing it is pure \
self-inflicted churn on a scope that can never be served again"
);
let counts_after = f.node.org_routing_reconciliation_counts();
assert_eq!(counts_after[2], counts_before[2], "and invalidate nothing");
assert_eq!(counts_after[0], counts_before[0], "and re-queue nothing");
drop(held);
let _ = f.node.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_second_installed_grant_still_withdraws_terminally_after_the_first() {
use crate::adapter::net::behavior::org_routing_registry::{GrantArtifactFence, SourceFacts};
let f = grant_fixture("ww16").await;
let (grant_a, secret_a) = f.mint("nrpc:ww16-a", Some([0xb1u8; 32]), None);
let (id_a, handle_a) = f.install(grant_a, secret_a);
let (grant_b, secret_b) = f.mint("nrpc:ww16-b", Some([0xb2u8; 32]), None);
let (id_b, handle_b) = f.install(grant_b, secret_b);
let key_a = f.key(id_a, handle_a, "nrpc:ww16-a");
let key_b = f.key(id_b, handle_b, "nrpc:ww16-b");
f.node.exhaust_consumer_grant_publications_for_test();
f.node.start();
assert!(until(|| f.node.org_routing_ready()).await, "healthy");
let family = f.node.org_routing_family().expect("family");
let held_a = family.demand(key_a.clone()).expect("demand a");
let held_b = family.demand(key_b.clone()).expect("demand b");
assert!(
until(|| {
[&key_a, &key_b].iter().all(|k| {
f.node
.routing_registry
.base_facts_unvalidated(k)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Served(_)))
})
})
.await,
"precondition: both grants are installed and both scopes are Served"
);
for (label, k) in [("A", &key_a), ("B", &key_b)] {
let facts = f
.node
.routing_registry
.base_facts_unvalidated(k)
.expect("warm");
assert!(
matches!(facts.grant_fence, GrantArtifactFence::Publication(_)),
"{label}: a scope with an INSTALLED grant carries an ordinary \
publication even though the identity space is spent. Exhaustion is \
a property of the space; terminal absence is a property of a \
reconstruction that found nothing"
);
}
let before_b = f
.node
.routing_registry
.base_facts_unvalidated(&key_b)
.expect("b");
assert!(
f.node.remove_consumer_grant_audience(&id_a),
"an exhausted space must not block revocation"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key_a)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"A's Served artifact must be retired and rebuilt absent"
);
let a_absent = f
.node
.routing_registry
.base_facts_unvalidated(&key_a)
.expect("a");
assert_eq!(
a_absent.grant_fence,
GrantArtifactFence::TerminalAbsence,
"A reconstructed absent under a spent space, so its absence is terminal"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key_b)
.is_some_and(|live| Arc::ptr_eq(&live, &before_b)),
"B keeps its EXACT artifact through A's terminal withdrawal"
);
assert!(
matches!(
f.node
.routing_registry
.base_facts_unvalidated(&key_b)
.expect("b")
.grant_fence,
GrantArtifactFence::Publication(_)
),
"and it is still an ORDINARY publication. Stamped TerminalAbsence here, \
B's own withdrawal below could not clear it — a grant that is gone \
would keep a Served artifact"
);
assert!(
f.node.remove_consumer_grant_audience(&id_b),
"a second terminal withdrawal must also be possible"
);
assert!(
until(|| {
f.node
.routing_registry
.base_facts_unvalidated(&key_b)
.is_some_and(|facts| matches!(facts.providers, SourceFacts::Unserved))
})
.await,
"B's Served artifact must be retired by B's OWN terminal withdrawal"
);
assert_eq!(
f.node
.routing_registry
.base_facts_unvalidated(&key_b)
.expect("b")
.grant_fence,
GrantArtifactFence::TerminalAbsence,
"and B rebuilds terminally absent in its turn"
);
assert!(
f.node
.routing_registry
.base_facts_unvalidated(&key_a)
.is_some_and(|live| Arc::ptr_eq(&live, &a_absent)),
"while A's terminal absence is untouched by it"
);
drop(held_a);
drop(held_b);
let _ = f.node.shutdown().await;
}
fn fresh_session_keys() -> crate::adapter::net::crypto::SessionKeys {
use crate::adapter::net::crypto::{NoiseHandshake, StaticKeypair};
let psk = [0x42u8; 32];
let responder_kp = StaticKeypair::generate();
let mut initiator =
NoiseHandshake::initiator(&psk, &responder_kp.public).expect("noise initiator");
let mut responder = NoiseHandshake::responder(&psk, &responder_kp).expect("noise responder");
let msg1 = initiator.write_message(&[]).expect("msg1");
responder.read_message(&msg1).expect("read msg1");
let msg2 = responder.write_message(&[]).expect("msg2");
initiator.read_message(&msg2).expect("read msg2");
initiator.into_session_keys().expect("session keys")
}
fn seed_peer(node: &MeshNode, node_id: u64, direct: bool) -> u64 {
let keys = fresh_session_keys();
let addr: SocketAddr = format!("10.9.0.{}:9000", (node_id % 250) + 1)
.parse()
.expect("addr");
let session = Arc::new(NetSession::new(keys, addr, 4, false));
let session_id = session.session_id();
node.peers.insert(
node_id,
PeerInfo {
node_id,
transport: if direct {
PeerTransport::Direct { owned_addr: addr }
} else {
PeerTransport::Routed {
relay_addr: addr,
adjacent_relay_identity: None,
}
},
session,
remote_static_pub: [0u8; 32],
last_initiator_ephemeral: None,
},
);
session_id
}
fn eligibility_of(
node: &MeshNode,
entity: &EntityId,
) -> crate::adapter::net::behavior::org_routing_registry::DirectEligibility {
use crate::adapter::net::behavior::org_routing_registry::SessionEligibility;
node.session_routing.published.load().eligibility(entity)
}
#[tokio::test]
async fn a_live_peer_session_annotates_the_provider_it_resolves_to() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let direct_entity = EntityId::from_bytes([0x21u8; 32]);
let relayed_entity = EntityId::from_bytes([0x22u8; 32]);
let sessionless = EntityId::from_bytes([0x23u8; 32]);
let direct_session = seed_peer(&node, 0x2001, true);
let relayed_session = seed_peer(&node, 0x2002, false);
node.test_pin_peer_entity(0x2001, direct_entity.clone());
node.test_pin_peer_entity(0x2002, relayed_entity.clone());
assert_eq!(
eligibility_of(&node, &direct_entity),
DirectEligibility::Direct {
node_id: 0x2001,
session_id: direct_session,
},
"an adjacent session is DIRECT and names the exact session it is"
);
assert_eq!(
eligibility_of(&node, &relayed_entity),
DirectEligibility::Relayed {
node_id: 0x2002,
session_id: relayed_session,
},
"a relayed session is annotated, and annotated as relayed — directness \
is a property of the transport, not of having a session at all"
);
assert_eq!(
eligibility_of(&node, &sessionless),
DirectEligibility::Cold,
"and an entity with no session is COLD"
);
}
#[tokio::test]
async fn an_entity_claimed_by_two_live_sessions_is_never_annotated() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let contested = EntityId::from_bytes([0x31u8; 32]);
let clean = EntityId::from_bytes([0x32u8; 32]);
seed_peer(&node, 0x3001, true);
seed_peer(&node, 0x3002, true);
seed_peer(&node, 0x3003, true);
node.test_pin_peer_entity(0x3001, contested.clone());
node.test_pin_peer_entity(0x3002, contested.clone());
node.test_pin_peer_entity(0x3003, clean.clone());
assert_eq!(
eligibility_of(&node, &contested),
DirectEligibility::Cold,
"two live sessions claim this provider: there is no determinate \
session, so there is no annotation"
);
assert!(
matches!(
eligibility_of(&node, &clean),
DirectEligibility::Direct {
node_id: 0x3003,
..
}
),
"while an unambiguous provider in the same projection is annotated"
);
}
#[tokio::test]
async fn a_pin_without_a_live_session_annotates_nothing() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let orphan = EntityId::from_bytes([0x41u8; 32]);
node.test_pin_peer_entity(0x4001, orphan.clone());
assert_eq!(
eligibility_of(&node, &orphan),
DirectEligibility::Cold,
"a mapping is not a session"
);
seed_peer(&node, 0x4002, true);
node.note_session_transition();
assert_eq!(
node.session_routing.published.load().rows.len(),
0,
"an unpinned session names no provider, so it annotates none"
);
}
#[tokio::test]
async fn a_republication_moves_the_projection_and_its_generation_together() {
let node = node().await;
let entity = EntityId::from_bytes([0x51u8; 32]);
seed_peer(&node, 0x5001, true);
let before = node.org_routing_session_publications();
node.test_pin_peer_entity(0x5001, entity.clone());
let after_pin = node.org_routing_session_publications();
assert_eq!(after_pin, before + 1, "a real pin publishes exactly once");
let published = node.session_routing.published.load_full();
assert_eq!(
Some(published.generation),
node.org_routing_session_generation(),
"the generation names the projection that is actually visible"
);
assert_eq!(published.rows.len(), 1);
node.test_pin_peer_entity(0x5001, entity.clone());
let republished = node.session_routing.published.load_full();
assert_eq!(
republished.rows, published.rows,
"an idempotent pin cannot change the projection's content"
);
assert!(
republished.generation >= published.generation,
"and the generation is monotone across it"
);
}
#[tokio::test]
async fn a_session_transition_retires_the_pool_and_the_actor_republishes_it() {
let node = node().await;
node.start();
assert!(
until(|| node.org_routing_ready()).await,
"precondition: the actor must reach Healthy before anything is demanded"
);
let key = slot(60, "nrpc:s2e2e");
let family = node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| node
.routing_registry
.unsensed_pool_unvalidated(&key)
.is_some())
.await,
"the production actor publishes a pool beside the facts it built"
);
let facts = node.org_routing_base_facts(&key).expect("facts");
let pool = node
.org_routing_unsensed_pool(&key)
.expect("validated pool");
assert!(
pool.derives_from(&facts),
"the validated read proves the pairing rather than assuming it"
);
let generation = pool.session_generation();
assert_eq!(
Some(generation),
node.org_routing_session_generation(),
"and the pool was computed under the LIVE session view"
);
let _ = quiesced_counts(&node).await;
let before = node.org_routing_pool_counts();
seed_peer(&node, 0x6001, true);
node.test_pin_peer_entity(0x6001, EntityId::from_bytes([0x61u8; 32]));
let moved = node
.org_routing_session_generation()
.expect("a live generation");
assert!(moved > generation, "the transition advanced the generation");
assert!(
until(|| node
.routing_registry
.unsensed_pool_unvalidated(&key)
.is_some_and(|p| p.session_generation() == moved))
.await,
"the actor republishes the pool under the new session view"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"and the discovery facts were preserved throughout — session movement \
is not a reason to re-reconstruct the source"
);
let _ = quiesced_counts(&node).await;
let after = node.org_routing_pool_counts();
assert_eq!(
after[1] - before[1],
1,
"the transition retired EXACTLY the one superseded pool this node holds"
);
assert_eq!(
after[0] - before[0],
1,
"and the actor republished EXACTLY one pool for it"
);
assert_eq!(
after[2] - before[2],
0,
"with no build refused: the rebuild ran after the projection settled"
);
drop(held);
let _ = node.shutdown().await;
}
#[tokio::test]
async fn the_validated_pool_read_refuses_a_mispaired_or_superseded_pool() {
let node = node().await;
node.start();
assert!(until(|| node.org_routing_ready()).await, "precondition");
seed_peer(&node, 0x6101, true);
node.test_pin_peer_entity(0x6101, EntityId::from_bytes([0x62u8; 32]));
let key = slot(61, "nrpc:s2read");
let family = node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| node.org_routing_unsensed_pool(&key).is_some()).await,
"precondition: a validated pool must exist to be refused"
);
let facts = node.org_routing_base_facts(&key).expect("facts");
let live_generation = node
.org_routing_session_generation()
.expect("a live generation");
let foreign = Arc::new(
crate::adapter::net::behavior::org_routing_registry::ScopedUnsensedRoutePool::for_test_at_session(
Arc::new((*facts).clone()),
live_generation,
),
);
assert!(
!foreign.derives_from(&facts),
"precondition: this pool names a DIFFERENT artifact…"
);
assert_eq!(
foreign.session_generation(),
live_generation,
"…while carrying the live session view, so only the pairing check can \
refuse it"
);
node.routing_registry
.install_unsensed_pool_for_test(&key, foreign);
assert!(
node.org_routing_unsensed_pool(&key).is_none(),
"a pool that does not name the published facts is COLD"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"and refusing it invalidates nothing — the facts are current"
);
let superseded = Arc::new(
crate::adapter::net::behavior::org_routing_registry::ScopedUnsensedRoutePool::for_test(
facts.clone(),
),
);
assert!(
superseded.derives_from(&facts),
"precondition: this pool DOES name the published facts, so only the \
session check can refuse it"
);
assert_ne!(
Some(superseded.session_generation()),
node.org_routing_session_generation(),
"precondition: and it was computed under an older view"
);
node.routing_registry
.install_unsensed_pool_for_test(&key, superseded);
assert!(
node.org_routing_unsensed_pool(&key).is_none(),
"a pool from a superseded session view is COLD"
);
assert!(
node.org_routing_base_facts(&key).is_some(),
"and refusing THAT invalidates nothing either"
);
drop(held);
let _ = node.shutdown().await;
}
#[tokio::test]
async fn an_exhausted_session_generation_fences_the_pool_plane() {
let node = node().await;
node.start();
assert!(until(|| node.org_routing_ready()).await, "precondition");
let key = slot(62, "nrpc:s2spent");
let family = node.org_routing_family().expect("family");
let held = family.demand(key.clone()).expect("demand");
assert!(
until(|| node.org_routing_unsensed_pool(&key).is_some()).await,
"precondition: a pool must exist for the exhaustion to retire"
);
node.session_routing.currentness.set_for_test(u64::MAX - 1);
let publications_before = node.org_routing_session_publications();
let retired_before = node.org_routing_pool_counts()[1];
seed_peer(&node, 0x7001, true);
node.test_pin_peer_entity(0x7001, EntityId::from_bytes([0x71u8; 32]));
assert_eq!(
node.org_routing_session_generation(),
None,
"the space is terminally spent"
);
assert_eq!(
node.org_routing_session_publications(),
publications_before,
"a transition that cannot allocate a generation must publish no \
projection at all"
);
assert!(
until(|| node
.routing_registry
.unsensed_pool_unvalidated(&key)
.is_none())
.await,
"every retained pool is retired once, rather than left published under \
a view nothing can witness"
);
let _ = quiesced_counts(&node).await;
assert_eq!(
node.org_routing_pool_counts()[1] - retired_before,
1,
"exactly ONE retirement for the one pool this node held; a spinning \
terminal arm would keep counting them"
);
assert!(
node.org_routing_unsensed_pool(&key).is_none(),
"and the validated read is cold"
);
assert!(
until(|| node.org_routing_base_facts(&key).is_some()).await,
"discovery is unaffected: the facts plane keeps reconciling"
);
let published_before = node.org_routing_pool_counts()[0];
let _ = quiesced_counts(&node).await;
let (_retained, pending) = node.org_routing_slots();
assert_eq!(
pending, 0,
"the actor parks rather than spinning on a plane it can never publish"
);
assert_eq!(
node.org_routing_pool_counts()[0],
published_before,
"and nothing is published under a spent generation"
);
drop(held);
let _ = node.shutdown().await;
}
#[tokio::test]
async fn a_lost_peer_install_publishes_no_session_generation() {
let node = node().await;
let addr: SocketAddr = "10.9.9.1:9000".parse().expect("addr");
let other: SocketAddr = "10.9.9.2:9000".parse().expect("addr");
let before = node.org_routing_session_publications();
let installed = node.install_direct(0x8001, addr, fresh_session_keys(), None);
assert!(
installed.owned,
"precondition: this transition owned the peer"
);
assert_eq!(
node.org_routing_session_publications(),
before + 1,
"a transition that published a session publishes a generation"
);
let generation = node.org_routing_session_generation();
let published = node.org_routing_session_publications();
let lost = node.install_direct(
0x8001,
other,
fresh_session_keys(),
Some(0xdead_beef_dead_beef),
);
assert!(!lost.owned, "precondition: the compare-and-swap must lose");
assert_eq!(
node.org_routing_session_publications(),
published,
"a non-publishing transition publishes no session generation"
);
assert_eq!(
node.org_routing_session_generation(),
generation,
"and the live generation does not move"
);
}
#[tokio::test]
async fn a_peer_replaced_mid_build_cannot_publish_its_old_session() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let entity = EntityId::from_bytes([0x81u8; 32]);
const N: u64 = 0x8001;
let s0 = seed_peer(&node, N, true);
node.test_pin_peer_entity(N, entity.clone());
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s0,
},
"precondition: the projection names S0"
);
let replaced: Arc<parking_lot::Mutex<Option<u64>>> = Arc::new(parking_lot::Mutex::new(None));
{
let node_for_hook = node.clone();
let replaced = replaced.clone();
node.session_routing
.observe_build_for_test(Arc::new(move |sampled| {
if sampled != N {
return;
}
let s1 = seed_peer(&node_for_hook, N, true);
*replaced.lock() = Some(s1);
}));
}
node.note_session_transition();
let s1 = replaced
.lock()
.take()
.expect("the interleaving must have run");
assert_ne!(s1, s0, "precondition: the replacement is a NEW incarnation");
let published = node.session_routing.published.load_full();
let row = published.rows.get(&entity).copied();
assert!(
!matches!(row, Some(DirectEligibility::Direct { session_id, .. })
| Some(DirectEligibility::Relayed { session_id, .. })
if session_id == s0),
"a build that sampled S0 must not publish it once S1 is installed; \
published {row:?} with S0 = {s0}"
);
assert!(
!matches!(eligibility_of(&node, &entity),
DirectEligibility::Direct { session_id, .. }
| DirectEligibility::Relayed { session_id, .. } if session_id == s0),
"and the eligibility a pool would be annotated from names no S0 either"
);
node.note_session_transition();
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s1,
},
"the rebuild after the replacement names the LIVE incarnation"
);
}
#[tokio::test]
async fn a_stable_peer_observed_twice_still_publishes_its_row() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let entity = EntityId::from_bytes([0x91u8; 32]);
const N: u64 = 0x9001;
let s0 = seed_peer(&node, N, true);
node.test_pin_peer_entity(N, entity.clone());
let observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let observed = observed.clone();
node.session_routing
.observe_build_for_test(Arc::new(move |sampled| {
if sampled == N {
observed.store(true, Ordering::Release);
}
}));
}
node.note_session_transition();
assert!(
observed.load(Ordering::Acquire),
"precondition: the interleaving point must actually have been reached, \
or this control proves nothing"
);
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s0,
},
"a peer that did not move is annotated with the session it has"
);
}
#[tokio::test]
async fn a_projection_and_its_generation_are_never_observed_torn() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
type Rows = std::collections::BTreeMap<EntityId, DirectEligibility>;
let node = node().await;
let entity = EntityId::from_bytes([0xa1u8; 32]);
const N: u64 = 0xa001;
seed_peer(&node, N, true);
node.test_pin_peer_entity(N, entity.clone());
let arriving = EntityId::from_bytes([0xa2u8; 32]);
const M: u64 = 0xa002;
node.test_pin_peer_entity(M, arriving.clone());
seed_peer(&node, M, false);
type Samples = Arc<parking_lot::Mutex<Vec<(&'static str, u64, Rows)>>>;
let samples: Samples = Arc::new(parking_lot::Mutex::new(Vec::new()));
let sample = {
let node_for_hook = node.clone();
let samples = samples.clone();
move |where_: &'static str| {
let seen = node_for_hook.session_routing.published.load_full();
samples
.lock()
.push((where_, seen.generation, seen.rows.clone()));
}
};
{
let sample = sample.clone();
node.session_routing
.observe_build_for_test(Arc::new(move |_| sample("mid-build")));
}
{
let sample = sample.clone();
node.session_routing
.observe_revalidated_for_test(Arc::new(move || sample("pre-store")));
}
let before = node.session_routing.published.load_full();
node.note_session_transition();
let after = node.session_routing.published.load_full();
assert_ne!(
after.rows, before.rows,
"precondition: this republication must actually change the row set, or \
old and new are indistinguishable and the witness proves nothing"
);
let samples = samples.lock().clone();
let mut seen_where: Vec<&str> = samples.iter().map(|(w, _, _)| *w).collect();
seen_where.sort_unstable();
seen_where.dedup();
assert_eq!(
seen_where,
vec!["mid-build", "pre-store"],
"both observation points must have been reached, or this witness proves \
nothing"
);
for (where_, generation, rows) in samples {
assert_eq!(
(generation, &rows),
(before.generation, &before.rows),
"a reader at {where_} observed neither the old projection nor the \
old generation: the two do not travel together"
);
}
assert!(
after.generation > before.generation,
"and the republication did advance the generation"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_send_in_flight_retains_no_peer_shard() {
let node = node().await;
const N: u64 = 0xb001;
let _s0 = seed_peer(&node, N, true);
let wrote = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let node_for_hook = node.clone();
let wrote = wrote.clone();
node.observe_send_guard_release_for_test(Arc::new(move || {
let node_for_thread = node_for_hook.clone();
let writer = std::thread::spawn(move || {
seed_peer(&node_for_thread, N, false);
});
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while !writer.is_finished() {
if std::time::Instant::now() >= deadline {
panic!(
"a peer-map WRITE could not proceed while a send was in \
flight: the send is retaining its peer shard across the \
await"
);
}
std::thread::sleep(Duration::from_millis(2));
}
writer.join().expect("writer thread");
wrote.store(true, Ordering::Release);
}));
}
let sent = node
.send_subprotocol_to_node(N, SUBPROTOCOL_CAPABILITY_ANN, b"probe")
.await;
assert!(
wrote.load(Ordering::Acquire),
"the observation point must have been reached, or this witness proves \
nothing"
);
let _ = sent;
assert!(
tokio::time::timeout(
Duration::from_secs(10),
node.announce_capabilities(
crate::adapter::net::behavior::capability::CapabilitySet::new()
)
)
.await
.is_ok(),
"AnnounceCapabilities must not be wedged behind a completed send"
);
}
fn flip_peer_transport(node: &MeshNode, node_id: u64) -> bool {
let Some(existing) = node.peers.get(&node_id).map(|p| {
(
p.session.clone(),
p.transport,
p.remote_static_pub,
p.last_initiator_ephemeral,
)
}) else {
return false;
};
let (session, transport, remote_static_pub, last_initiator_ephemeral) = existing;
let addr = transport.send_addr();
let flipped = match transport {
PeerTransport::Direct { .. } => PeerTransport::Routed {
relay_addr: addr,
adjacent_relay_identity: None,
},
PeerTransport::Routed { .. } => PeerTransport::Direct { owned_addr: addr },
};
node.peers.insert(
node_id,
PeerInfo {
node_id,
transport: flipped,
session,
remote_static_pub,
last_initiator_ephemeral,
},
);
true
}
#[tokio::test]
async fn a_transport_flipped_mid_build_cannot_publish_its_old_directness() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let entity = EntityId::from_bytes([0xc1u8; 32]);
const N: u64 = 0xc001;
let s0 = seed_peer(&node, N, true);
node.test_pin_peer_entity(N, entity.clone());
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s0,
},
"precondition: the projection names a DIRECT S0"
);
let flipped = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let node_for_hook = node.clone();
let flipped = flipped.clone();
node.session_routing
.observe_build_for_test(Arc::new(move |sampled| {
if sampled != N {
return;
}
assert!(
flip_peer_transport(&node_for_hook, N),
"the interleaving must actually have flipped the transport"
);
flipped.store(true, Ordering::Release);
}));
}
node.note_session_transition();
assert!(
flipped.load(Ordering::Acquire),
"precondition: the interleaving must have run"
);
let published = node.session_routing.published.load_full();
assert_ne!(
published.rows.get(&entity).copied(),
Some(DirectEligibility::Direct {
node_id: N,
session_id: s0,
}),
"a build that sampled DIRECT must not publish it once the transport is \
relayed, even though the session id never moved"
);
node.note_session_transition();
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Relayed {
node_id: N,
session_id: s0,
},
"and the rebuild annotates it RELAYED, on the same session"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peer_snapshot_retains_no_shard_and_drops_the_ingress_peer() {
let node = node().await;
const A: u64 = 0xe001;
const B: u64 = 0xe002;
let _session_a = seed_peer(&node, A, true);
let _session_b = seed_peer(&node, B, false);
let write_attempt = || -> (bool, std::thread::JoinHandle<()>) {
let node = node.clone();
let writer = std::thread::spawn(move || {
seed_peer(&node, A, false);
});
let until = std::time::Instant::now() + Duration::from_millis(300);
while !writer.is_finished() {
if std::time::Instant::now() >= until {
return (true, writer);
}
std::thread::sleep(Duration::from_millis(2));
}
(false, writer)
};
let control = {
let held: Vec<_> = node.peers.iter().collect();
assert!(!held.is_empty(), "precondition: the iterator holds guards");
let (blocked, writer) = write_attempt();
assert!(
blocked,
"a retained peer-map iterator must block a writer — without this the \
positive half below proves nothing"
);
drop(held);
writer
};
control.join().expect("control writer thread");
let snapshot = snapshot_peers(&node.peers, Some(B));
assert_eq!(
snapshot.len(),
1,
"the ingress peer is excluded and every other peer is present"
);
let owned_session = snapshot[0].session.session_id();
let (blocked, writer) = write_attempt();
assert!(!blocked, "a live snapshot must retain no peer shard");
writer.join().expect("writer thread");
assert_ne!(
node.peers
.get(&A)
.map(|p| p.session.session_id())
.expect("peer A"),
owned_session,
"precondition: that write REPLACED the session the snapshot holds"
);
assert_eq!(
snapshot[0].session.session_id(),
owned_session,
"and the snapshot owns the session it was taken with — the replacement \
cannot retroactively change what this fan-out is sending on"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn the_announce_lock_is_acquired_under_a_bound_and_refuses_past_it() {
let node = node().await;
let held = node.announce_mu.lock();
let started = std::time::Instant::now();
assert!(
node.lock_announce_mu_within(Duration::from_millis(150))
.is_none(),
"an acquire that cannot complete within its bound must REFUSE"
);
let waited = started.elapsed();
assert!(
waited < Duration::from_secs(5),
"and it must refuse promptly — waited {waited:?}"
);
drop(held);
let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
let holder = {
let node = node.clone();
std::thread::spawn(move || {
let guard = node.announce_mu.lock();
acquired_tx.send(()).expect("signal");
std::thread::sleep(Duration::from_millis(100));
drop(guard);
})
};
acquired_rx.recv().expect("the holder must have acquired");
assert!(
node.lock_announce_mu_within(Duration::from_secs(10))
.is_some(),
"a lock released inside the bound must be acquired, not refused"
);
holder.join().expect("holder thread");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peer_replaced_after_revalidation_cannot_publish_its_old_session() {
use crate::adapter::net::behavior::org_routing_registry::DirectEligibility;
let node = node().await;
let entity = EntityId::from_bytes([0xd1u8; 32]);
const N: u64 = 0xd001;
let addr: SocketAddr = "10.9.8.1:9000".parse().expect("addr");
let replacement_addr: SocketAddr = "10.9.8.2:9000".parse().expect("addr");
assert!(
node.install_direct(N, addr, fresh_session_keys(), None)
.owned,
"precondition: the seed install owned the peer"
);
node.test_pin_peer_entity(N, entity.clone());
let s0 = node
.peers
.get(&N)
.map(|p| p.session.session_id())
.expect("the seeded peer");
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s0,
},
"precondition: the projection names S0"
);
let fired = Arc::new(std::sync::atomic::AtomicBool::new(false));
let stable = Arc::new(std::sync::atomic::AtomicBool::new(false));
let blocked = Arc::new(std::sync::atomic::AtomicBool::new(false));
let at_gate = Arc::new(std::sync::atomic::AtomicBool::new(false));
#[allow(clippy::type_complexity)]
let replacer: Arc<parking_lot::Mutex<Option<std::thread::JoinHandle<bool>>>> =
Arc::new(parking_lot::Mutex::new(None));
{
let node_for_hook = node.clone();
let (fired, stable, blocked, at_gate, replacer) = (
fired.clone(),
stable.clone(),
blocked.clone(),
at_gate.clone(),
replacer.clone(),
);
node.session_routing
.observe_revalidated_for_test(Arc::new(move || {
fired.store(true, Ordering::Release);
let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel::<()>(1);
node_for_hook
.session_routing
.observe_transition_attempt_for_test(Arc::new(move || {
let _ = reached_tx.try_send(());
}));
let node_for_thread = node_for_hook.clone();
let handle = std::thread::spawn(move || {
node_for_thread
.install_direct(N, replacement_addr, fresh_session_keys(), None)
.owned
});
at_gate.store(
reached_rx.recv_timeout(Duration::from_secs(10)).is_ok(),
Ordering::Release,
);
let until = std::time::Instant::now() + Duration::from_millis(300);
let mut held = true;
while std::time::Instant::now() < until {
if node_for_hook.peers.get(&N).map(|p| p.session.session_id()) != Some(s0) {
held = false;
break;
}
std::thread::sleep(Duration::from_millis(5));
}
stable.store(held, Ordering::Release);
blocked.store(!handle.is_finished(), Ordering::Release);
*replacer.lock() = Some(handle);
}));
}
let publications_before = node.org_routing_session_publications();
let generation_before = node
.org_routing_session_generation()
.expect("a live generation");
node.note_session_transition();
assert!(
fired.load(Ordering::Acquire),
"precondition: the revalidated→published window must have been reached"
);
assert!(
at_gate.load(Ordering::Acquire),
"the replacement never reached the pre-lock seam ahead of \
`session_routing.gate.lock()` — without that, `!is_finished()` below \
would be satisfied by a thread that had merely not started yet"
);
assert!(
stable.load(Ordering::Acquire),
"a peer replacement landed between the build's revalidation and its \
store: the projection about to be published names a session `peers` no \
longer holds"
);
assert!(
blocked.load(Ordering::Acquire),
"the replacement must have been BLOCKED for the whole window, not \
merely late"
);
let handle = replacer.lock().take().expect("the replacement thread");
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while !handle.is_finished() {
assert!(
std::time::Instant::now() < deadline,
"the replacement never proceeded after the window closed"
);
tokio::time::sleep(Duration::from_millis(5)).await;
}
assert!(
handle.join().expect("replacement thread"),
"the replacement transition owned the peer"
);
let s1 = node
.peers
.get(&N)
.map(|p| p.session.session_id())
.expect("the replaced peer");
assert_ne!(s1, s0, "precondition: the replacement is a NEW incarnation");
assert_eq!(
node.org_routing_session_publications(),
publications_before + 2,
"exactly two publications, in order: this build, then the replacement's \
own — which is what makes the generation naming S1 strictly newer than \
the one naming S0"
);
assert_eq!(
node.org_routing_session_generation(),
Some(generation_before + 2),
"and the generation advanced once per publication"
);
assert_eq!(
eligibility_of(&node, &entity),
DirectEligibility::Direct {
node_id: N,
session_id: s1,
},
"the live projection names the LIVE incarnation"
);
}