use std::cell::{Ref, RefCell};
use std::collections::{HashMap, HashSet};
use std::rc::Rc;
use std::time::Instant;
use super::supervisor::auth_adapter::broker_report::{
AuthenticatedBrokerTraceReport, BrokerResumeSender,
};
use super::supervisor::{ConnectionIdentity, ValidatedSpawn};
const MAX_LIVE_SESSIONS: usize = 64;
const MAX_SESSIONS_PER_SERVICE_GENERATION: usize = 4096;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum WatchdogStateError {
CapacityExceeded,
UnknownSession,
WrongConnection,
InvalidTransition,
DeadlineExpired,
BrokerActivationFailed,
BrokerResumeFailed,
}
#[derive(Debug, Eq, PartialEq)]
pub(super) struct FreshSessionId([u8; 32]);
impl FreshSessionId {
pub(super) unsafe fn from_fresh_random(value: [u8; 32]) -> Result<Self, WatchdogStateError> {
if value == [0; 32] {
Err(WatchdogStateError::UnknownSession)
} else {
Ok(Self(value))
}
}
pub(super) const fn handle(&self) -> SessionHandle {
SessionHandle(self.0)
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub(super) struct SessionHandle([u8; 32]);
impl SessionHandle {
pub(super) const fn bytes(self) -> [u8; 32] {
self.0
}
}
struct RegisteredLaunch<Launch> {
handle: SessionHandle,
connection: ConnectionIdentity,
deadline: Instant,
launch: Launch,
}
pub(super) struct AtomicallySpawnedBroker<Launch, Authority: ExactBrokerAuthority, Report = ()> {
session: FreshSessionId,
launch: Launch,
broker: ExactBroker<Authority>,
report: Report,
}
impl<Launch, Authority: ExactBrokerAuthority> AtomicallySpawnedBroker<Launch, Authority> {
#[cfg(test)]
pub(super) unsafe fn from_test_atomic_spawn(
session: FreshSessionId,
launch: Launch,
broker: ExactBroker<Authority>,
) -> Self {
Self {
session,
launch,
broker,
report: (),
}
}
}
#[cfg(test)]
impl<Launch, Authority: ExactBrokerAuthority, Report>
AtomicallySpawnedBroker<Launch, Authority, Report>
{
pub(super) unsafe fn from_test_atomic_spawn_with_report(
session: FreshSessionId,
launch: Launch,
broker: ExactBroker<Authority>,
report: Report,
) -> Self {
Self {
session,
launch,
broker,
report,
}
}
}
impl<Launch, Authority: ExactBrokerAuthority, Report>
AtomicallySpawnedBroker<Launch, Authority, Report>
{
fn into_parts(self) -> (FreshSessionId, Launch, ExactBroker<Authority>, Report) {
(self.session, self.launch, self.broker, self.report)
}
}
impl
AtomicallySpawnedBroker<
ValidatedSpawn,
super::supervisor::auth_adapter::broker_spawn::DirectChildBrokerAuthority,
super::supervisor::auth_adapter::broker_report::BrokerTraceReportReceiver,
>
{
pub(super) fn from_fixed_image_spawn(
spawned: super::supervisor::auth_adapter::broker_spawn::FixedImageBrokerSpawn,
) -> Self {
let (session, launch, broker, report) = spawned.into_parts();
Self {
session,
launch,
broker,
report,
}
}
}
impl<Launch> RegisteredLaunch<Launch> {
const fn handle(&self) -> SessionHandle {
self.handle
}
const fn connection(&self) -> ConnectionIdentity {
self.connection
}
const fn deadline(&self) -> Instant {
self.deadline
}
}
struct RegisteredSession<Launch, Report = ()> {
handle: SessionHandle,
launch: RegisteredLaunch<Launch>,
report: Report,
}
impl<Launch, Report> RegisteredSession<Launch, Report> {
const fn handle(&self) -> SessionHandle {
self.handle
}
fn into_parts(self) -> (RegisteredLaunch<Launch>, Report) {
(self.launch, self.report)
}
}
pub(super) trait RegisteredLaunchEffect {
fn connection_identity(&self) -> ConnectionIdentity;
fn deadline(&self) -> Instant;
}
impl RegisteredLaunchEffect for ValidatedSpawn {
fn connection_identity(&self) -> ConnectionIdentity {
self.connection_identity()
}
fn deadline(&self) -> Instant {
self.deadline()
}
}
pub(super) struct ExactBroker<Authority: ExactBrokerAuthority> {
authority: Authority,
armed: bool,
}
impl<Authority: ExactBrokerAuthority> ExactBroker<Authority> {
pub(super) const unsafe fn from_unreaped_direct_child(authority: Authority) -> Self {
Self {
authority,
armed: true,
}
}
fn mark_reaped(&mut self, _proof: ReapedBroker) {
self.armed = false;
}
#[cfg(test)]
pub(super) fn authority_mut_for_test(&mut self) -> &mut Authority {
&mut self.authority
}
#[cfg(test)]
pub(super) fn mark_reaped_for_test(&mut self, proof: ReapedBroker) {
self.mark_reaped(proof);
}
}
impl<Authority: ExactBrokerAuthority> Drop for ExactBroker<Authority> {
fn drop(&mut self) {
if self.armed {
let proof = self.authority.emergency_terminate_and_reap(None);
self.mark_reaped(proof);
}
}
}
pub(super) unsafe trait ExactBrokerAuthority {
type Failure;
fn activate_after_registration(&mut self) -> Result<(), Self::Failure>;
fn terminate_and_reap(
&mut self,
reason: TerminationReason,
) -> Result<ReapedBroker, Self::Failure>;
fn emergency_terminate_and_reap(&mut self, reason: Option<TerminationReason>) -> ReapedBroker;
}
pub(super) struct ReapedBroker(());
impl ReapedBroker {
pub(super) const unsafe fn from_exact_reap() -> Self {
Self(())
}
}
pub(super) struct TraceEstablished {
handle: SessionHandle,
connection: ConnectionIdentity,
resume: Option<BrokerResumeSender>,
}
#[must_use = "a ready proof must be delivered or exact-cleaned"]
pub(super) struct ReadySessionProof {
handle: SessionHandle,
connection: ConnectionIdentity,
resume: Option<BrokerResumeSender>,
}
#[must_use = "Ready delivery must succeed or exact-clean the bound broker"]
pub(super) struct PendingReadyDelivery<Authority: ExactBrokerAuthority> {
entry: Rc<RefCell<WatchdogEntry<Authority>>>,
proof: Option<ReadySessionProof>,
cleanup_reason: TerminationReason,
}
pub(super) unsafe trait NonblockingReadySend {
type Error;
fn cleanup_reason(error: &Self::Error) -> TerminationReason;
fn send_once(
self,
handle: SessionHandle,
connection: ConnectionIdentity,
deadline: Instant,
) -> Result<(), Self::Error>;
}
impl<Authority: ExactBrokerAuthority> PendingReadyDelivery<Authority> {
pub(super) fn deliver<Send: NonblockingReadySend>(
self,
send: Send,
) -> Result<Result<(), Send::Error>, WatchdogStateError> {
self.deliver_at(Instant::now(), send)
}
fn deliver_at<Send: NonblockingReadySend>(
mut self,
now: Instant,
send: Send,
) -> Result<Result<(), Send::Error>, WatchdogStateError> {
let proof = self.proof.as_ref().expect("armed Ready proof");
let handle = proof.handle;
let connection = proof.connection;
let entry = self
.entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != handle || entry.connection != connection {
std::process::abort();
}
match entry.phase {
BrokerPhase::Reaped(_) => return Err(WatchdogStateError::UnknownSession),
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Starting
| BrokerPhase::Traced
| BrokerPhase::ReadyCommitted
| BrokerPhase::TerminationRequired(_) => {}
}
if entry.phase != BrokerPhase::Traced {
return Err(WatchdogStateError::InvalidTransition);
}
if now >= entry.deadline {
self.cleanup_reason = TerminationReason::DeadlineExpired;
return Err(WatchdogStateError::DeadlineExpired);
}
let deadline = entry.deadline;
drop(entry);
match send.send_once(handle, connection, deadline) {
Ok(()) => {
if let Some(resume) = self
.proof
.as_mut()
.expect("armed Ready proof")
.resume
.as_mut()
&& resume.commit_after_ready().is_err()
{
self.cleanup_reason = TerminationReason::ProtocolViolation;
return Err(WatchdogStateError::BrokerResumeFailed);
}
let mut entry = self
.entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != handle
|| entry.connection != connection
|| entry.phase != BrokerPhase::Traced
{
std::process::abort();
}
entry.phase = BrokerPhase::ReadyCommitted;
drop(entry);
self.proof.take();
Ok(Ok(()))
}
Err(error) => {
self.cleanup_reason = Send::cleanup_reason(&error);
Ok(Err(error))
}
}
}
}
impl<Authority: ExactBrokerAuthority> Drop for PendingReadyDelivery<Authority> {
fn drop(&mut self) {
if let Some(proof) = self.proof.take() {
emergency_terminate_entry(
&self.entry,
proof.handle,
proof.connection,
self.cleanup_reason,
);
}
}
}
impl ReadySessionProof {
pub(super) const fn handle(&self) -> SessionHandle {
self.handle
}
pub(super) const fn connection(&self) -> ConnectionIdentity {
self.connection
}
}
impl TraceEstablished {
pub(super) const fn handle(&self) -> SessionHandle {
self.handle
}
pub(super) const fn connection(&self) -> ConnectionIdentity {
self.connection
}
pub(super) fn from_authenticated_broker_report(report: AuthenticatedBrokerTraceReport) -> Self {
let (handle, connection, resume) = report.into_parts();
Self {
handle,
connection,
resume: Some(resume),
}
}
#[cfg(test)]
pub(super) const unsafe fn from_broker_handshake(
handle: SessionHandle,
connection: ConnectionIdentity,
) -> Self {
Self {
handle,
connection,
resume: None,
}
}
#[cfg(test)]
pub(super) const unsafe fn from_broker_handshake_with_resume(
handle: SessionHandle,
connection: ConnectionIdentity,
resume: BrokerResumeSender,
) -> Self {
Self {
handle,
connection,
resume: Some(resume),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BrokerPhase {
Starting,
Traced,
ReadyCommitted,
TerminationRequired(TerminationReason),
Reaping(TerminationReason),
Reaped(TerminationReason),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum TerminationReason {
LaunchAbandoned,
ClientDisconnected,
DeadlineExpired,
ClientRequested,
UnexpectedBrokerStop,
ProtocolViolation,
SpawnResultUndeliverable,
}
struct WatchdogEntry<Authority: ExactBrokerAuthority> {
handle: SessionHandle,
connection: ConnectionIdentity,
deadline: Instant,
broker: Option<ExactBroker<Authority>>,
phase: BrokerPhase,
}
pub(super) struct WatchdogTable<Authority: ExactBrokerAuthority> {
live: HashMap<SessionHandle, Rc<RefCell<WatchdogEntry<Authority>>>>,
tombstones: HashSet<SessionHandle>,
}
#[must_use = "a registered spawn must reach Ready or exact-clean its broker"]
pub(super) struct PendingRegisteredSession<Launch, Authority: ExactBrokerAuthority, Report = ()> {
entry: Rc<RefCell<WatchdogEntry<Authority>>>,
handle: SessionHandle,
connection: ConnectionIdentity,
launch: Option<RegisteredLaunch<Launch>>,
report: Option<Report>,
trace: Option<TraceEstablished>,
cleanup_reason: TerminationReason,
armed: bool,
}
pub(super) struct RegisteredLaunchPermit<'lease, Launch, Authority: ExactBrokerAuthority> {
entry: Rc<RefCell<WatchdogEntry<Authority>>>,
launch: &'lease RegisteredLaunch<Launch>,
}
#[must_use = "the launch commitment must remain live through irreversible exec"]
pub(super) struct RegisteredLaunchCommitGuard<'permit, Authority: ExactBrokerAuthority> {
_entry: Ref<'permit, WatchdogEntry<Authority>>,
}
impl<Launch, Authority: ExactBrokerAuthority> RegisteredLaunchPermit<'_, Launch, Authority> {
pub(super) const fn handle(&self) -> SessionHandle {
self.launch.handle()
}
pub(super) const fn connection(&self) -> ConnectionIdentity {
self.launch.connection()
}
pub(super) const fn deadline(&self) -> Instant {
self.launch.deadline()
}
pub(super) const fn launch(&self) -> &Launch {
&self.launch.launch
}
pub(super) fn commit_guard(
&self,
) -> Result<RegisteredLaunchCommitGuard<'_, Authority>, WatchdogStateError> {
let entry = self
.entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != self.launch.handle || entry.connection != self.launch.connection {
std::process::abort();
}
match entry.phase {
BrokerPhase::Starting => {}
BrokerPhase::Reaped(_) => return Err(WatchdogStateError::UnknownSession),
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Traced
| BrokerPhase::ReadyCommitted
| BrokerPhase::TerminationRequired(_) => {
return Err(WatchdogStateError::InvalidTransition);
}
}
if Instant::now() >= entry.deadline {
drop(entry);
emergency_terminate_entry(
&self.entry,
self.launch.handle,
self.launch.connection,
TerminationReason::DeadlineExpired,
);
return Err(WatchdogStateError::DeadlineExpired);
}
Ok(RegisteredLaunchCommitGuard { _entry: entry })
}
}
impl<Launch, Authority: ExactBrokerAuthority, Report>
PendingRegisteredSession<Launch, Authority, Report>
{
pub(super) const fn handle(&self) -> SessionHandle {
self.handle
}
pub(super) const fn connection(&self) -> ConnectionIdentity {
self.connection
}
pub(super) fn launch_permit(
&self,
) -> Result<RegisteredLaunchPermit<'_, Launch, Authority>, WatchdogStateError> {
let entry = self
.entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != self.handle || entry.connection != self.connection {
std::process::abort();
}
if entry.phase != BrokerPhase::Starting {
return Err(WatchdogStateError::InvalidTransition);
}
if Instant::now() >= entry.deadline {
drop(entry);
emergency_terminate_entry(
&self.entry,
self.handle,
self.connection,
TerminationReason::DeadlineExpired,
);
return Err(WatchdogStateError::DeadlineExpired);
}
let launch = self
.launch
.as_ref()
.ok_or(WatchdogStateError::InvalidTransition)?;
Ok(RegisteredLaunchPermit {
entry: Rc::clone(&self.entry),
launch,
})
}
pub(super) fn report_mut(&mut self) -> Result<&mut Report, WatchdogStateError> {
if !self.armed {
std::process::abort();
}
let entry = self
.entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != self.handle || entry.connection != self.connection {
std::process::abort();
}
if entry.phase != BrokerPhase::Starting {
return Err(WatchdogStateError::InvalidTransition);
}
if Instant::now() >= entry.deadline {
drop(entry);
self.cleanup_reason = TerminationReason::DeadlineExpired;
return Err(WatchdogStateError::DeadlineExpired);
}
drop(entry);
self.report
.as_mut()
.ok_or(WatchdogStateError::InvalidTransition)
}
pub(super) fn consume_report(&mut self) -> Result<Report, WatchdogStateError> {
if !self.armed {
std::process::abort();
}
let entry = self
.entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != self.handle || entry.connection != self.connection {
std::process::abort();
}
if entry.phase != BrokerPhase::Starting {
return Err(WatchdogStateError::InvalidTransition);
}
if Instant::now() >= entry.deadline {
drop(entry);
self.cleanup_reason = TerminationReason::DeadlineExpired;
return Err(WatchdogStateError::DeadlineExpired);
}
drop(entry);
self.report
.take()
.ok_or(WatchdogStateError::InvalidTransition)
}
pub(super) fn mark_protocol_violation(&mut self) {
self.cleanup_reason = TerminationReason::ProtocolViolation;
}
pub(super) fn mark_deadline_expired(&mut self) {
self.cleanup_reason = TerminationReason::DeadlineExpired;
}
pub(super) fn bind_trace(&mut self, trace: TraceEstablished) -> Result<(), TraceEstablished> {
if self.trace.is_some()
|| trace.handle != self.handle
|| trace.connection != self.connection
{
return Err(trace);
}
self.trace = Some(trace);
Ok(())
}
pub(super) fn mark_traced_for_delivery(
mut self,
) -> Result<PendingReadyDelivery<Authority>, WatchdogStateError> {
let Some(trace) = self.trace.take() else {
self.mark_protocol_violation();
return Err(WatchdogStateError::InvalidTransition);
};
self.armed = false;
transition_registered_for_delivery(Rc::clone(&self.entry), trace)
}
}
impl<Launch, Authority: ExactBrokerAuthority, Report> Drop
for PendingRegisteredSession<Launch, Authority, Report>
{
fn drop(&mut self) {
if self.armed {
emergency_terminate_entry(
&self.entry,
self.handle,
self.connection,
self.cleanup_reason,
);
self.armed = false;
}
drop(self.report.take());
}
}
fn transition_registered_for_delivery<Authority: ExactBrokerAuthority>(
entry: Rc<RefCell<WatchdogEntry<Authority>>>,
proof: TraceEstablished,
) -> Result<PendingReadyDelivery<Authority>, WatchdogStateError> {
let transition_error = {
let mut entry_ref = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry_ref.handle != proof.handle || entry_ref.connection != proof.connection {
std::process::abort();
}
if entry_ref.phase != BrokerPhase::Starting {
Some(WatchdogStateError::InvalidTransition)
} else if Instant::now() >= entry_ref.deadline {
Some(WatchdogStateError::DeadlineExpired)
} else {
entry_ref.phase = BrokerPhase::Traced;
None
}
};
if let Some(error) = transition_error {
emergency_terminate_entry(
&entry,
proof.handle,
proof.connection,
if error == WatchdogStateError::DeadlineExpired {
TerminationReason::DeadlineExpired
} else {
TerminationReason::ProtocolViolation
},
);
return Err(error);
}
Ok(PendingReadyDelivery {
entry,
proof: Some(ReadySessionProof {
handle: proof.handle,
connection: proof.connection,
resume: proof.resume,
}),
cleanup_reason: TerminationReason::SpawnResultUndeliverable,
})
}
fn emergency_terminate_entry<Authority: ExactBrokerAuthority>(
entry: &Rc<RefCell<WatchdogEntry<Authority>>>,
handle: SessionHandle,
connection: ConnectionIdentity,
reason: TerminationReason,
) {
let (mut broker, effective_reason) = {
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.handle != handle || entry.connection != connection {
std::process::abort();
}
let effective_reason = match entry.phase {
BrokerPhase::Reaped(_) => return,
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Starting | BrokerPhase::Traced | BrokerPhase::ReadyCommitted => reason,
BrokerPhase::TerminationRequired(existing) => existing,
};
entry.phase = BrokerPhase::Reaping(effective_reason);
(
entry.broker.take().unwrap_or_else(|| std::process::abort()),
effective_reason,
)
};
let proof = broker
.authority
.emergency_terminate_and_reap(Some(effective_reason));
broker.mark_reaped(proof);
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.phase != BrokerPhase::Reaping(effective_reason) || entry.broker.is_some() {
std::process::abort();
}
entry.phase = BrokerPhase::Reaped(effective_reason);
}
impl<Authority: ExactBrokerAuthority> WatchdogTable<Authority> {
pub(super) fn new() -> Self {
Self {
live: HashMap::new(),
tombstones: HashSet::new(),
}
}
fn register<Launch: RegisteredLaunchEffect, Report>(
&mut self,
spawned: AtomicallySpawnedBroker<Launch, Authority, Report>,
) -> Result<RegisteredSession<Launch, Report>, WatchdogStateError> {
self.sweep_reaped();
let (session, launch, broker, report) = spawned.into_parts();
if self.live.len() >= MAX_LIVE_SESSIONS
|| self.live.len().saturating_add(self.tombstones.len())
>= MAX_SESSIONS_PER_SERVICE_GENERATION
{
return Err(WatchdogStateError::CapacityExceeded);
}
let handle = SessionHandle(session.0);
let connection = launch.connection_identity();
let deadline = launch.deadline();
if self.live.contains_key(&handle) || self.tombstones.contains(&handle) {
return Err(WatchdogStateError::UnknownSession);
}
self.live.insert(
handle,
Rc::new(RefCell::new(WatchdogEntry {
handle,
connection,
deadline,
broker: Some(broker),
phase: BrokerPhase::Starting,
})),
);
if Instant::now() >= deadline {
let entry = Rc::clone(
self.live
.get(&handle)
.expect("expired registration retains the exact session entry"),
);
emergency_terminate_entry(
&entry,
handle,
connection,
TerminationReason::DeadlineExpired,
);
self.sweep_reaped();
drop(report);
return Err(WatchdogStateError::DeadlineExpired);
}
let activation = self
.live
.get(&handle)
.expect("registration inserted the exact session entry")
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort())
.broker
.as_mut()
.unwrap_or_else(|| std::process::abort())
.authority
.activate_after_registration();
if activation.is_err() {
let entry = Rc::clone(
self.live
.get(&handle)
.expect("failed activation retains the exact session entry"),
);
emergency_terminate_entry(
&entry,
handle,
connection,
TerminationReason::LaunchAbandoned,
);
self.sweep_reaped();
drop(report);
return Err(WatchdogStateError::BrokerActivationFailed);
}
Ok(RegisteredSession {
handle,
launch: RegisteredLaunch {
handle,
connection,
deadline,
launch,
},
report,
})
}
pub(super) fn register_armed<Launch: RegisteredLaunchEffect, Report>(
&mut self,
spawned: AtomicallySpawnedBroker<Launch, Authority, Report>,
) -> Result<PendingRegisteredSession<Launch, Authority, Report>, WatchdogStateError> {
let registered = self.register(spawned)?;
let handle = registered.handle();
let (launch, report) = registered.into_parts();
let connection = launch.connection();
let entry = Rc::clone(
self.live
.get(&handle)
.expect("registration inserted the exact session entry"),
);
Ok(PendingRegisteredSession {
entry,
handle,
connection,
launch: Some(launch),
report: Some(report),
trace: None,
cleanup_reason: TerminationReason::LaunchAbandoned,
armed: true,
})
}
fn mark_traced(
&mut self,
proof: TraceEstablished,
) -> Result<ReadySessionProof, WatchdogStateError> {
let entry = self.client_entry(proof.handle, proof.connection)?;
let expired = {
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.phase != BrokerPhase::Starting {
return Err(WatchdogStateError::InvalidTransition);
}
if Instant::now() >= entry.deadline {
true
} else {
entry.phase = BrokerPhase::Traced;
false
}
};
if expired {
let _ = self.terminate_internal(proof.handle, TerminationReason::DeadlineExpired);
return Err(WatchdogStateError::DeadlineExpired);
}
Ok(ReadySessionProof {
handle: proof.handle,
connection: proof.connection,
resume: proof.resume,
})
}
pub(super) fn mark_traced_for_delivery(
&mut self,
proof: TraceEstablished,
) -> Result<PendingReadyDelivery<Authority>, WatchdogStateError> {
let entry = self.client_entry(proof.handle, proof.connection)?;
transition_registered_for_delivery(entry, proof)
}
pub(super) fn terminate_undelivered_ready(
&mut self,
proof: ReadySessionProof,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
let entry = self.client_entry(proof.handle, proof.connection)?;
if entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort())
.phase
!= BrokerPhase::Traced
{
return Err(WatchdogStateError::InvalidTransition);
}
self.terminate_internal(proof.handle, TerminationReason::SpawnResultUndeliverable)
}
pub(super) fn terminate_for_client_disconnect(
&mut self,
handle: SessionHandle,
connection: ConnectionIdentity,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.terminate_for_client(handle, connection, TerminationReason::ClientDisconnected)
}
pub(super) fn terminate_for_client_request(
&mut self,
handle: SessionHandle,
connection: ConnectionIdentity,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.terminate_for_client(handle, connection, TerminationReason::ClientRequested)
}
pub(super) fn terminate_for_deadline(
&mut self,
handle: SessionHandle,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.sweep_reaped();
let entry = self
.live
.get(&handle)
.ok_or(WatchdogStateError::UnknownSession)?;
let (phase, deadline) = {
let entry = entry.try_borrow().unwrap_or_else(|_| std::process::abort());
(entry.phase, entry.deadline)
};
match phase {
BrokerPhase::ReadyCommitted => return Err(WatchdogStateError::InvalidTransition),
BrokerPhase::Reaped(_) => return Err(WatchdogStateError::UnknownSession),
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Starting | BrokerPhase::Traced | BrokerPhase::TerminationRequired(_) => {}
}
if Instant::now() < deadline {
return Err(WatchdogStateError::InvalidTransition);
}
self.terminate_internal(handle, TerminationReason::DeadlineExpired)
}
pub(super) fn terminate_for_unexpected_stop(
&mut self,
handle: SessionHandle,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.terminate_internal(handle, TerminationReason::UnexpectedBrokerStop)
}
pub(super) fn terminate_for_protocol_violation(
&mut self,
handle: SessionHandle,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.terminate_internal(handle, TerminationReason::ProtocolViolation)
}
fn terminate_for_client(
&mut self,
handle: SessionHandle,
connection: ConnectionIdentity,
reason: TerminationReason,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.client_entry(handle, connection)?;
self.terminate_internal(handle, reason)
}
fn terminate_internal(
&mut self,
handle: SessionHandle,
reason: TerminationReason,
) -> Result<Result<(), Authority::Failure>, WatchdogStateError> {
self.sweep_reaped();
let entry = Rc::clone(
self.live
.get(&handle)
.ok_or(WatchdogStateError::UnknownSession)?,
);
let (mut broker, effective_reason) = {
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
let effective_reason = match entry.phase {
BrokerPhase::Starting | BrokerPhase::Traced | BrokerPhase::ReadyCommitted => reason,
BrokerPhase::TerminationRequired(existing) => existing,
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Reaped(_) => return Err(WatchdogStateError::UnknownSession),
};
entry.phase = BrokerPhase::Reaping(effective_reason);
let broker = entry.broker.take().unwrap_or_else(|| std::process::abort());
(broker, effective_reason)
};
let result = broker.authority.terminate_and_reap(effective_reason);
let result = match result {
Ok(proof) => {
broker.mark_reaped(proof);
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.phase != BrokerPhase::Reaping(effective_reason) || entry.broker.is_some() {
std::process::abort();
}
entry.phase = BrokerPhase::Reaped(effective_reason);
Ok(())
}
Err(error) => {
let mut entry = entry
.try_borrow_mut()
.unwrap_or_else(|_| std::process::abort());
if entry.phase != BrokerPhase::Reaping(effective_reason) || entry.broker.is_some() {
std::process::abort();
}
entry.phase = BrokerPhase::TerminationRequired(effective_reason);
entry.broker = Some(broker);
Err(error)
}
};
if result.is_ok() {
self.sweep_reaped();
}
Ok(result)
}
fn emergency_terminate_registered(
&mut self,
handle: SessionHandle,
connection: ConnectionIdentity,
reason: TerminationReason,
) {
let Ok(entry) = self.client_entry(handle, connection) else {
std::process::abort();
};
emergency_terminate_entry(&entry, handle, connection, reason);
self.sweep_reaped();
}
fn client_entry(
&mut self,
handle: SessionHandle,
connection: ConnectionIdentity,
) -> Result<Rc<RefCell<WatchdogEntry<Authority>>>, WatchdogStateError> {
self.sweep_reaped();
let entry = Rc::clone(
self.live
.get(&handle)
.ok_or(WatchdogStateError::UnknownSession)?,
);
let entry_connection = entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort())
.connection;
if entry_connection != connection {
return Err(WatchdogStateError::WrongConnection);
}
Ok(entry)
}
fn sweep_reaped(&mut self) {
let reaped: Vec<_> = self
.live
.iter()
.filter_map(|(handle, entry)| {
let phase = entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort())
.phase;
match phase {
BrokerPhase::Reaped(_) => Some(*handle),
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Starting
| BrokerPhase::Traced
| BrokerPhase::ReadyCommitted
| BrokerPhase::TerminationRequired(_) => None,
}
})
.collect();
for handle in reaped {
if !self.tombstones.insert(handle) {
std::process::abort();
}
self.live
.remove(&handle)
.unwrap_or_else(|| std::process::abort());
}
}
#[cfg(test)]
fn contains_live(&self, handle: SessionHandle) -> bool {
self.live.get(&handle).is_some_and(|entry| {
matches!(
entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort())
.phase,
BrokerPhase::Starting
| BrokerPhase::Traced
| BrokerPhase::ReadyCommitted
| BrokerPhase::TerminationRequired(_)
)
})
}
#[cfg(test)]
fn contains_tombstone(&self, handle: SessionHandle) -> bool {
self.tombstones.contains(&handle)
|| self.live.get(&handle).is_some_and(|entry| {
matches!(
entry
.try_borrow()
.unwrap_or_else(|_| std::process::abort())
.phase,
BrokerPhase::Reaped(_)
)
})
}
}
impl<Authority: ExactBrokerAuthority> Drop for WatchdogTable<Authority> {
fn drop(&mut self) {
for entry in self.live.values() {
let (handle, connection, phase) = {
let entry = entry.try_borrow().unwrap_or_else(|_| std::process::abort());
(entry.handle, entry.connection, entry.phase)
};
match phase {
BrokerPhase::Reaped(_) => {}
BrokerPhase::Reaping(_) => std::process::abort(),
BrokerPhase::Starting
| BrokerPhase::Traced
| BrokerPhase::ReadyCommitted
| BrokerPhase::TerminationRequired(_) => emergency_terminate_entry(
entry,
handle,
connection,
TerminationReason::ProtocolViolation,
),
}
}
}
}
#[cfg(test)]
#[path = "supervisor_watchdog_test.rs"]
mod tests;