use std::io::{Read, Write};
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::thread::{self, JoinHandle};
use std::time::Duration;
use crate::ServerError;
use crate::server::listener::{loopback_interrupt_target, shed_on_fd_exhaustion};
use super::checks::{SharedReadinessState, health_check, readiness_check};
use super::reissue::{
OperatorCredentialReissueOutcome, OperatorCredentialReissueRefusal,
OperatorCredentialReissueRequest, OperatorCredentialReissuer, SharedOperatorCredentialReissue,
};
use super::unloadable::{SharedUnloadableConversations, UnloadableConversationRecord};
use super::metrics_route;
const HEALTH_PATH: &str = "/health";
const READY_PATH: &str = "/ready";
const METRICS_PATH: &str = "/metrics";
const UNLOADABLE_PATH: &str = "/unloadable-conversations";
const REISSUE_PATH: &str = "/operator/credential-reissue";
const APPLICATION_JSON: &str = "application/json";
const READ_BUFFER_BYTES: usize = 2048;
#[derive(Debug)]
pub struct HealthServerHandle {
local_addr: SocketAddr,
interrupt_target: SocketAddr,
shutdown: Arc<AtomicBool>,
active_stream: Arc<Mutex<Option<TcpStream>>>,
unloadable: SharedUnloadableConversations,
reissue: SharedOperatorCredentialReissue,
worker: Option<JoinHandle<Result<(), ServerError>>>,
#[cfg(test)]
accept_attempts: Arc<AtomicU64>,
#[cfg(test)]
shed_count: Arc<AtomicU64>,
#[cfg(test)]
requests_entered: Arc<AtomicU64>,
}
impl HealthServerHandle {
#[must_use]
pub const fn local_addr(&self) -> SocketAddr {
self.local_addr
}
pub fn install_unloadable_record(&self, record: UnloadableConversationRecord) {
self.unloadable.install(record);
}
pub fn install_credential_reissuer(&self, reissuer: Arc<dyn OperatorCredentialReissuer>) {
self.reissue.install(reissuer);
}
pub fn shutdown(mut self) -> Result<(), ServerError> {
self.stop_worker()
}
fn stop_worker(&mut self) -> Result<(), ServerError> {
self.shutdown.store(true, Ordering::SeqCst);
let Some(worker) = self.worker.take() else {
return Ok(());
};
let in_flight = self
.active_stream
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take();
if let Some(stream) = in_flight {
let _ = stream.shutdown(Shutdown::Both);
}
if let Ok(waker) = TcpStream::connect(self.interrupt_target) {
drop(waker);
}
worker.join().map_err(|_| ServerError::HealthEndpoint {
message: "health endpoint worker thread terminated unexpectedly".to_owned(),
})?
}
#[cfg(test)]
fn accept_attempts(&self) -> u64 {
self.accept_attempts.load(Ordering::SeqCst)
}
#[cfg(test)]
fn shed_count(&self) -> u64 {
self.shed_count.load(Ordering::SeqCst)
}
#[cfg(test)]
fn requests_entered(&self) -> u64 {
self.requests_entered.load(Ordering::SeqCst)
}
}
impl Drop for HealthServerHandle {
fn drop(&mut self) {
if let Err(error) = self.stop_worker() {
tracing::debug!(%error, "health endpoint shutdown during drop failed");
}
}
}
pub fn start_health_server(
bind_address: SocketAddr,
readiness: SharedReadinessState,
) -> Result<HealthServerHandle, ServerError> {
let listener =
TcpListener::bind(bind_address).map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to bind health endpoint at {bind_address}: {error}"),
})?;
let local_addr = listener
.local_addr()
.map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to inspect health endpoint listener address: {error}"),
})?;
let interrupt_target = loopback_interrupt_target(local_addr);
let shutdown = Arc::new(AtomicBool::new(false));
let active_stream = Arc::new(Mutex::new(None));
let unloadable = SharedUnloadableConversations::default();
let reissue = SharedOperatorCredentialReissue::default();
let accept_attempts = Arc::new(AtomicU64::new(0));
let shed_count = Arc::new(AtomicU64::new(0));
let requests_entered = Arc::new(AtomicU64::new(0));
let worker_shutdown = Arc::clone(&shutdown);
let worker_stream = Arc::clone(&active_stream);
let worker_attempts = Arc::clone(&accept_attempts);
let worker_shed = Arc::clone(&shed_count);
let worker_entered = Arc::clone(&requests_entered);
let served = ServedState {
readiness,
unloadable: unloadable.clone(),
reissue: reissue.clone(),
};
let worker = thread::spawn(move || {
serve(
&listener,
&served,
&worker_shutdown,
&worker_stream,
&worker_attempts,
&worker_shed,
&worker_entered,
)
});
Ok(HealthServerHandle {
local_addr,
interrupt_target,
shutdown,
active_stream,
unloadable,
reissue,
worker: Some(worker),
#[cfg(test)]
accept_attempts,
#[cfg(test)]
shed_count,
#[cfg(test)]
requests_entered,
})
}
#[derive(Debug, Clone)]
struct ServedState {
readiness: SharedReadinessState,
unloadable: SharedUnloadableConversations,
reissue: SharedOperatorCredentialReissue,
}
fn serve(
listener: &TcpListener,
served: &ServedState,
shutdown: &AtomicBool,
active_stream: &Mutex<Option<TcpStream>>,
accept_attempts: &AtomicU64,
shed_count: &AtomicU64,
requests_entered: &AtomicU64,
) -> Result<(), ServerError> {
let mut reserve = listener.try_clone().ok();
while !shutdown.load(Ordering::SeqCst) {
accept_attempts.fetch_add(1, Ordering::SeqCst);
match listener.accept() {
Ok((stream, ..)) => {
let admitted = {
let mut slot = active_stream.lock().unwrap_or_else(PoisonError::into_inner);
if shutdown.load(Ordering::SeqCst) {
false
} else {
*slot = stream.try_clone().ok();
true
}
};
if !admitted {
drop(stream);
continue;
}
requests_entered.fetch_add(1, Ordering::SeqCst);
let result = handle_connection(stream, served);
*active_stream.lock().unwrap_or_else(PoisonError::into_inner) = None;
if let Err(error) = result {
tracing::debug!(%error, "health endpoint connection error");
}
}
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
Err(error) if is_transient_accept_error(&error) => {
shed_on_fd_exhaustion(listener, &mut reserve, shed_count, &error);
}
Err(error) => {
return Err(ServerError::HealthEndpoint {
message: format!("health endpoint accept failed: {error}"),
});
}
}
}
Ok(())
}
fn is_transient_accept_error(error: &std::io::Error) -> bool {
matches!(error.raw_os_error(), Some(code) if code == 24 || code == 23)
}
fn handle_connection(mut stream: TcpStream, served: &ServedState) -> Result<(), ServerError> {
stream
.set_nonblocking(false)
.map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to configure health request stream: {error}"),
})?;
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to set health request read timeout: {error}"),
})?;
let mut buffer = [0_u8; READ_BUFFER_BYTES];
let bytes_read = stream
.read(&mut buffer)
.map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to read health request: {error}"),
})?;
if bytes_read == 0 {
return Ok(());
}
let response = response_for_request(&buffer[..bytes_read], served)?;
stream
.write_all(&response)
.map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to write health response: {error}"),
})?;
stream.flush().map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to flush health response: {error}"),
})
}
fn response_for_request(request: &[u8], served: &ServedState) -> Result<Vec<u8>, ServerError> {
let Ok(request) = std::str::from_utf8(request) else {
return Ok(empty_response(StatusCode::BadRequest));
};
let Some((method, path)) = parse_request_line(request) else {
return Ok(empty_response(StatusCode::BadRequest));
};
match (method, path) {
("GET", HEALTH_PATH) => json_response(StatusCode::Ok, &health_check()),
("GET", READY_PATH) => {
let status = readiness_check(&served.readiness.snapshot());
let status_code = if status.ready {
StatusCode::Ok
} else {
StatusCode::ServiceUnavailable
};
json_response(status_code, &status)
}
("GET", METRICS_PATH) => Ok(response(
StatusCode::Ok,
Some(metrics_route::CONTENT_TYPE),
metrics_route::render_body().as_bytes(),
)),
("GET", UNLOADABLE_PATH) => json_response(StatusCode::Ok, &served.unloadable.status()),
(_, HEALTH_PATH | READY_PATH | METRICS_PATH | UNLOADABLE_PATH) => {
Ok(empty_response(StatusCode::MethodNotAllowed))
}
_ if path.split('?').next() == Some(REISSUE_PATH) => {
credential_reissue_response(method, path, served)
}
_ => Ok(empty_response(StatusCode::NotFound)),
}
}
fn credential_reissue_response(
method: &str,
path: &str,
served: &ServedState,
) -> Result<Vec<u8>, ServerError> {
if method != "POST" {
return Ok(empty_response(StatusCode::MethodNotAllowed));
}
let query = path.split_once('?').map_or("", |(_, query)| query);
let Some(request) = parse_reissue_query(query) else {
return Ok(empty_response(StatusCode::BadRequest));
};
match served.reissue.reissue(request) {
Ok(None) => Ok(empty_response(StatusCode::ServiceUnavailable)),
Ok(Some(OperatorCredentialReissueOutcome::Issued(issued))) => {
json_response(StatusCode::Ok, &issued)
}
Ok(Some(OperatorCredentialReissueOutcome::Refused(refusal))) => {
json_response(reissue_refusal_status(&refusal), &refusal)
}
Err(error) => {
tracing::error!(%error, "operator credential re-issue could not be decided");
json_response(
StatusCode::ServiceUnavailable,
&serde_json::json!({ "error": error.message }),
)
}
}
}
const fn reissue_refusal_status(refusal: &OperatorCredentialReissueRefusal) -> StatusCode {
match refusal {
OperatorCredentialReissueRefusal::ConversationUnknown { .. }
| OperatorCredentialReissueRefusal::ParticipantUnknown { .. } => StatusCode::NotFound,
OperatorCredentialReissueRefusal::Retired { .. }
| OperatorCredentialReissueRefusal::LiveBinding { .. }
| OperatorCredentialReissueRefusal::DetachReplayOpen { .. }
| OperatorCredentialReissueRefusal::LiveReceipt { .. }
| OperatorCredentialReissueRefusal::GenerationMismatch { .. } => StatusCode::Conflict,
}
}
fn parse_reissue_query(query: &str) -> Option<OperatorCredentialReissueRequest> {
let mut conversation_id = None;
let mut participant_id = None;
let mut expected_current_generation = None;
for pair in query.split('&') {
let (name, value) = pair.split_once('=')?;
let value = value.parse::<u64>().ok()?;
let slot = match name {
"conversation_id" => &mut conversation_id,
"participant_id" => &mut participant_id,
"expected_current_generation" => &mut expected_current_generation,
_ => return None,
};
if slot.replace(value).is_some() {
return None;
}
}
Some(OperatorCredentialReissueRequest {
conversation_id: conversation_id?,
participant_id: participant_id?,
expected_current_generation: expected_current_generation?,
})
}
fn parse_request_line(request: &str) -> Option<(&str, &str)> {
let request_line = request.lines().next()?;
let mut parts = request_line.split_whitespace();
let method = parts.next()?;
let path = parts.next()?;
parts.next()?;
Some((method, path))
}
fn json_response<T>(status: StatusCode, value: &T) -> Result<Vec<u8>, ServerError>
where
T: serde::Serialize,
{
let body = serde_json::to_vec(value).map_err(|error| ServerError::HealthEndpoint {
message: format!("failed to serialize health response: {error}"),
})?;
Ok(response(status, Some(APPLICATION_JSON), &body))
}
fn empty_response(status: StatusCode) -> Vec<u8> {
response(status, None, &[])
}
fn response(status: StatusCode, content_type: Option<&str>, body: &[u8]) -> Vec<u8> {
let mut response = Vec::new();
let status_line = format!("HTTP/1.1 {} {}\r\n", status.code(), status.reason());
response.extend_from_slice(status_line.as_bytes());
response.extend_from_slice(format!("Content-Length: {}\r\n", body.len()).as_bytes());
response.extend_from_slice(b"Connection: close\r\n");
if let Some(content_type) = content_type {
response.extend_from_slice(format!("Content-Type: {content_type}\r\n").as_bytes());
}
response.extend_from_slice(b"\r\n");
response.extend_from_slice(body);
response
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StatusCode {
Ok,
BadRequest,
NotFound,
MethodNotAllowed,
Conflict,
ServiceUnavailable,
}
impl StatusCode {
const fn code(self) -> u16 {
match self {
Self::Ok => 200,
Self::BadRequest => 400,
Self::NotFound => 404,
Self::MethodNotAllowed => 405,
Self::Conflict => 409,
Self::ServiceUnavailable => 503,
}
}
const fn reason(self) -> &'static str {
match self {
Self::Ok => "OK",
Self::BadRequest => "Bad Request",
Self::NotFound => "Not Found",
Self::MethodNotAllowed => "Method Not Allowed",
Self::Conflict => "Conflict",
Self::ServiceUnavailable => "Service Unavailable",
}
}
}
#[cfg(test)]
mod tests {
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::sync::{Arc, Mutex, PoisonError};
use std::thread;
use std::time::{Duration, Instant};
use serde_json::Value;
use super::{
OperatorCredentialReissueRefusal, OperatorCredentialReissueRequest,
OperatorCredentialReissuer, ServedState, response_for_request, start_health_server,
};
use crate::health::checks::{
ClusterReadiness, ReadinessCondition, ReadinessState, SharedReadinessState,
};
fn loopback_ephemeral() -> Result<SocketAddr, Box<dyn std::error::Error>> {
Ok("127.0.0.1:0".parse()?)
}
fn served(readiness: SharedReadinessState) -> ServedState {
ServedState {
readiness,
unloadable: crate::health::unloadable::SharedUnloadableConversations::default(),
reissue: crate::health::reissue::SharedOperatorCredentialReissue::default(),
}
}
fn get(address: SocketAddr, path: &str) -> Result<String, Box<dyn std::error::Error>> {
let mut stream = TcpStream::connect(address)?;
stream.set_read_timeout(Some(Duration::from_secs(2)))?;
let request = format!("GET {path} HTTP/1.1\r\nHost: localhost\r\n\r\n");
stream.write_all(request.as_bytes())?;
let mut response = String::new();
stream.read_to_string(&mut response)?;
Ok(response)
}
fn assert_status(response: &str, status: u16) {
let expected = format!("HTTP/1.1 {status} ");
assert!(
response.starts_with(&expected),
"response status did not start with {expected}: {response}"
);
}
fn body(response: &str) -> Result<&str, Box<dyn std::error::Error>> {
let Some((_headers, body)) = response.split_once("\r\n\r\n") else {
return Err("response did not contain a header/body separator".into());
};
Ok(body)
}
fn json_body(response: &str) -> Result<Value, Box<dyn std::error::Error>> {
Ok(serde_json::from_str(body(response)?)?)
}
#[test]
fn health_endpoint_returns_json_200_regardless_of_readiness()
-> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::new(ReadinessState::default());
let server = start_health_server(loopback_ephemeral()?, readiness)?;
let response = get(server.local_addr(), "/health")?;
server.shutdown()?;
assert_status(&response, 200);
assert!(response.contains("Content-Type: application/json\r\n"));
let body = json_body(&response)?;
assert_eq!(body["status"], "healthy");
Ok(())
}
#[test]
fn ready_endpoint_returns_503_before_main_listener_binds()
-> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::new(ReadinessState::new(
true,
false,
ClusterReadiness::NotConfigured,
));
let server = start_health_server(loopback_ephemeral()?, readiness)?;
let response = get(server.local_addr(), "/ready")?;
server.shutdown()?;
assert_status(&response, 503);
assert!(response.contains("Content-Type: application/json\r\n"));
let body = json_body(&response)?;
assert_eq!(body["ready"], false);
assert_eq!(body["unmet_conditions"][0], "listener_bound");
Ok(())
}
#[test]
fn ready_endpoint_returns_200_after_all_startup_gates() -> Result<(), Box<dyn std::error::Error>>
{
let readiness = SharedReadinessState::new(ReadinessState::ready_without_cluster());
let server = start_health_server(loopback_ephemeral()?, readiness)?;
let response = get(server.local_addr(), "/ready")?;
server.shutdown()?;
assert_status(&response, 200);
let body = json_body(&response)?;
assert_eq!(body["ready"], true);
let Some(unmet_conditions) = body["unmet_conditions"].as_array() else {
return Err("unmet_conditions should be an array".into());
};
assert!(unmet_conditions.is_empty());
Ok(())
}
#[test]
fn ready_endpoint_updates_from_shared_readiness_state() -> Result<(), Box<dyn std::error::Error>>
{
let readiness = SharedReadinessState::new(ReadinessState::default());
let server = start_health_server(loopback_ephemeral()?, readiness.clone())?;
let response = get(server.local_addr(), "/ready")?;
assert_status(&response, 503);
readiness.set_config_loaded(true);
readiness.set_listener_bound(true);
let response = get(server.local_addr(), "/ready")?;
server.shutdown()?;
assert_status(&response, 200);
Ok(())
}
#[test]
fn clustered_ready_transitions_503_to_200_when_membership_established()
-> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::new(ReadinessState::new(
true,
true,
ClusterReadiness::Configured {
membership_established: false,
},
));
let server = start_health_server(loopback_ephemeral()?, readiness.clone())?;
let response = get(server.local_addr(), "/ready")?;
assert_status(&response, 503);
let body = json_body(&response)?;
assert_eq!(body["ready"], false);
assert_eq!(
body["unmet_conditions"][0],
serde_json::to_value(ReadinessCondition::ClusterMembershipEstablished)?
);
readiness.set_cluster_membership_established(true);
let response = get(server.local_addr(), "/ready")?;
server.shutdown()?;
assert_status(&response, 200);
let body = json_body(&response)?;
assert_eq!(body["ready"], true);
let Some(unmet_conditions) = body["unmet_conditions"].as_array() else {
return Err("unmet_conditions should be an array".into());
};
assert!(unmet_conditions.is_empty());
Ok(())
}
#[test]
fn cluster_readiness_is_listed_when_configured_but_not_joined()
-> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::new(ReadinessState::new(
true,
true,
ClusterReadiness::Configured {
membership_established: false,
},
));
let response = response_for_request(b"GET /ready HTTP/1.1\r\n\r\n", &served(readiness))?;
let response = String::from_utf8(response)?;
assert_status(&response, 503);
let body = json_body(&response)?;
assert_eq!(
body["unmet_conditions"][0],
serde_json::to_value(ReadinessCondition::ClusterMembershipEstablished)?
);
Ok(())
}
#[test]
fn unloadable_conversations_route_answers_the_operator_a_json_shape()
-> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::new(ReadinessState::default());
let server = start_health_server(loopback_ephemeral()?, readiness)?;
let response = get(server.local_addr(), "/unloadable-conversations")?;
let unknown = get(server.local_addr(), "/unloadable-conversations-typo")?;
server.shutdown()?;
assert_status(&unknown, 404);
assert_status(&response, 200);
assert!(
response.contains("Content-Type: application/json\r\n"),
"the unloadable-conversations route must answer JSON: {response}"
);
let body = json_body(&response)?;
assert_eq!(
body["count"], 0,
"a server with no participant record attached refuses nothing: {body}"
);
assert_eq!(
body["participant_installed"], false,
"no participant record is attached to this server, and the surface must say so \
rather than let a zero count read as a clean node: {body}"
);
let Some(conversations) = body["conversations"].as_array() else {
return Err("conversations should be an array".into());
};
assert!(
conversations.is_empty(),
"no conversation was refused: {conversations:?}"
);
Ok(())
}
#[test]
fn unsupported_paths_are_not_served() -> Result<(), Box<dyn std::error::Error>> {
let readiness = SharedReadinessState::default();
let response = response_for_request(b"GET /unknown HTTP/1.1\r\n\r\n", &served(readiness))?;
let response = String::from_utf8(response)?;
assert_status(&response, 404);
Ok(())
}
#[test]
fn unsupported_methods_on_health_paths_are_rejected() -> Result<(), Box<dyn std::error::Error>>
{
let readiness = SharedReadinessState::default();
let response = response_for_request(b"POST /health HTTP/1.1\r\n\r\n", &served(readiness))?;
let response = String::from_utf8(response)?;
assert_status(&response, 405);
Ok(())
}
#[derive(Debug)]
struct RecordingReissuer {
outcome: crate::health::reissue::OperatorCredentialReissueOutcome,
seen: Mutex<Vec<OperatorCredentialReissueRequest>>,
}
impl OperatorCredentialReissuer for RecordingReissuer {
fn reissue(
&self,
request: OperatorCredentialReissueRequest,
) -> Result<
crate::health::reissue::OperatorCredentialReissueOutcome,
crate::health::reissue::OperatorCredentialReissueError,
> {
self.seen
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(request);
Ok(self.outcome.clone())
}
}
fn served_with_reissuer(
outcome: crate::health::reissue::OperatorCredentialReissueOutcome,
) -> (ServedState, Arc<RecordingReissuer>) {
let reissuer = Arc::new(RecordingReissuer {
outcome,
seen: Mutex::new(Vec::new()),
});
let state = served(SharedReadinessState::default());
state
.reissue
.install(Arc::clone(&reissuer) as Arc<dyn OperatorCredentialReissuer>);
(state, reissuer)
}
const REISSUE_TARGET: &str =
"/operator/credential-reissue?conversation_id=7&participant_id=3&\
expected_current_generation=14";
#[test]
fn the_reissue_route_reports_an_uninstalled_participant()
-> Result<(), Box<dyn std::error::Error>> {
let request = format!("POST {REISSUE_TARGET} HTTP/1.1\r\n\r\n");
let response =
response_for_request(request.as_bytes(), &served(SharedReadinessState::default()))?;
let response = String::from_utf8(response)?;
assert_status(&response, 503);
Ok(())
}
#[test]
fn the_reissue_route_carries_the_three_inputs_and_returns_the_secret_once()
-> Result<(), Box<dyn std::error::Error>> {
let issued = crate::health::reissue::OperatorCredentialReissued {
conversation_id: 7,
participant_id: 3,
presented_generation: 14,
issued_generation: 15,
attach_secret: crate::health::reissue::encode_hex(&[0x5A; 32]),
};
let (state, reissuer) = served_with_reissuer(
crate::health::reissue::OperatorCredentialReissueOutcome::Issued(issued),
);
let request = format!("POST {REISSUE_TARGET} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 200);
let body = json_body(&response)?;
assert_eq!(body["issued_generation"], 15);
assert_eq!(body["presented_generation"], 14);
assert_eq!(body["attach_secret"], "5a".repeat(32));
let seen = reissuer
.seen
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
assert_eq!(
seen.as_slice(),
[OperatorCredentialReissueRequest {
conversation_id: 7,
participant_id: 3,
expected_current_generation: 14,
}]
);
Ok(())
}
#[test]
fn the_reissue_route_serves_the_normative_generation_pair()
-> Result<(), Box<dyn std::error::Error>> {
let (state, _) = served_with_reissuer(
crate::health::reissue::OperatorCredentialReissueOutcome::Refused(
OperatorCredentialReissueRefusal::GenerationMismatch {
conversation_id: 7,
participant_id: 3,
presented_generation: 14,
current_generation: 15,
},
),
);
let request = format!("POST {REISSUE_TARGET} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 409);
let body = json_body(&response)?;
assert_eq!(body["refusal"], "generation_mismatch");
assert_eq!(body["presented_generation"], 14);
assert_eq!(body["current_generation"], 15);
Ok(())
}
#[test]
fn the_reissue_route_answers_a_lookup_miss_with_not_found()
-> Result<(), Box<dyn std::error::Error>> {
let (state, _) = served_with_reissuer(
crate::health::reissue::OperatorCredentialReissueOutcome::Refused(
OperatorCredentialReissueRefusal::ConversationUnknown { conversation_id: 7 },
),
);
let request = format!("POST {REISSUE_TARGET} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 404);
let body = json_body(&response)?;
assert_eq!(body["refusal"], "conversation_unknown");
assert_eq!(body["conversation_id"], 7);
assert!(
body.get("participant_id").is_none(),
"an unknown-conversation refusal must disclose nothing beyond the presented \
conversation id: {body}"
);
Ok(())
}
#[test]
fn a_malformed_reissue_call_is_refused_and_never_reaches_the_authority()
-> Result<(), Box<dyn std::error::Error>> {
let (state, reissuer) = served_with_reissuer(
crate::health::reissue::OperatorCredentialReissueOutcome::Refused(
OperatorCredentialReissueRefusal::ConversationUnknown { conversation_id: 7 },
),
);
let malformed = [
"/operator/credential-reissue?conversation_id=7&participant_id=3",
"/operator/credential-reissue",
"/operator/credential-reissue?conversation_id=7&participant_id=3&\
expected_current_generation=fourteen",
"/operator/credential-reissue?conversation_id=7&participant_id=3&\
expected_current_generation=14&force=1",
"/operator/credential-reissue?conversation_id=7&conversation_id=8&\
participant_id=3&expected_current_generation=14",
];
for target in malformed {
let request = format!("POST {target} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 400);
}
assert!(
reissuer
.seen
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_empty(),
"a malformed call must never reach the serialized participant-state point"
);
Ok(())
}
#[test]
fn the_reissue_route_refuses_every_other_method() -> Result<(), Box<dyn std::error::Error>> {
let (state, reissuer) = served_with_reissuer(
crate::health::reissue::OperatorCredentialReissueOutcome::Refused(
OperatorCredentialReissueRefusal::ConversationUnknown { conversation_id: 7 },
),
);
for method in ["GET", "PUT", "DELETE"] {
let request = format!("{method} {REISSUE_TARGET} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 405);
}
assert!(
reissuer
.seen
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_empty()
);
Ok(())
}
#[test]
fn the_reissue_route_did_not_move_any_existing_routes_answer()
-> Result<(), Box<dyn std::error::Error>> {
let state = served(SharedReadinessState::default());
for target in [
"/health?x=1",
"/ready?x=1",
"/metrics?x=1",
"/unloadable-conversations?x=1",
] {
let request = format!("GET {target} HTTP/1.1\r\n\r\n");
let response = String::from_utf8(response_for_request(request.as_bytes(), &state)?)?;
assert_status(&response, 404);
}
let response =
String::from_utf8(response_for_request(b"GET /health HTTP/1.1\r\n\r\n", &state)?)?;
assert_status(&response, 200);
Ok(())
}
#[test]
fn silent_health_listener_has_zero_application_wakes() -> Result<(), Box<dyn std::error::Error>>
{
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let deadline = Instant::now() + Duration::from_secs(2);
while server.accept_attempts() < 1 && Instant::now() < deadline {
thread::sleep(Duration::from_millis(5));
}
let armed = server.accept_attempts();
assert_eq!(
armed, 1,
"the blocking accept is issued exactly once when parked"
);
thread::sleep(Duration::from_millis(200));
assert_eq!(
server.accept_attempts(),
armed,
"a silent health listener must not wake or re-accept"
);
assert_eq!(
server.shed_count(),
0,
"a silent health listener sheds nothing"
);
let response = get(server.local_addr(), "/health")?;
assert_status(&response, 200);
let body = json_body(&response)?;
assert_eq!(body["status"], "healthy");
server.shutdown()?;
Ok(())
}
#[test]
fn health_accept_source_has_no_wouldblock_sleep_poll() {
const SOURCE: &str = include_str!("endpoint.rs");
let production = SOURCE.split("mod tests").next().unwrap_or(SOURCE);
for forbidden in [
"set_nonblocking(true)",
"ErrorKind::WouldBlock",
"thread::sleep",
] {
assert!(
!production.contains(forbidden),
"retired health accept-path source `{forbidden}` reappeared"
);
}
}
#[test]
fn health_shutdown_interrupts_accept_wait() -> Result<(), Box<dyn std::error::Error>> {
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let start = Instant::now();
server.shutdown()?;
assert!(
start.elapsed() < Duration::from_secs(2),
"shutdown before arming must interrupt promptly, not sleep-poll"
);
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let deadline = Instant::now() + Duration::from_secs(2);
while server.accept_attempts() < 1 && Instant::now() < deadline {
thread::sleep(Duration::from_millis(5));
}
assert_eq!(
server.accept_attempts(),
1,
"the worker parked before shutdown"
);
let parked_addr = server.local_addr();
let start = Instant::now();
server.shutdown()?;
assert!(
start.elapsed() < Duration::from_secs(2),
"shutdown of a parked accept must interrupt promptly"
);
assert!(
TcpStream::connect(parked_addr).is_err(),
"the listener descriptor was released; further connects are refused"
);
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let deadline = Instant::now() + Duration::from_secs(2);
while server.accept_attempts() < 1 && Instant::now() < deadline {
thread::sleep(Duration::from_millis(5));
}
let _pending = TcpStream::connect(server.local_addr())?;
let start = Instant::now();
server.shutdown()?;
assert!(
start.elapsed() < Duration::from_millis(500),
"shutdown concurrent with a pending accept must interrupt promptly (TOLD), \
not defer by a request read deadline: elapsed {:?}",
start.elapsed()
);
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let response = get(server.local_addr(), "/health")?;
assert_status(&response, 200);
let start = Instant::now();
server.shutdown()?;
assert!(
start.elapsed() < Duration::from_secs(2),
"shutdown after a served request must interrupt the next parked accept promptly"
);
Ok(())
}
#[test]
fn shutdown_interrupts_in_flight_silent_request_read() -> Result<(), Box<dyn std::error::Error>>
{
let server = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let _silent = TcpStream::connect(server.local_addr())?;
let deadline = Instant::now() + Duration::from_secs(2);
while server.requests_entered() < 1 && Instant::now() < deadline {
thread::sleep(Duration::from_millis(5));
}
assert_eq!(
server.requests_entered(),
1,
"the worker entered the in-flight silent request read"
);
let start = Instant::now();
server.shutdown()?;
assert!(
start.elapsed() < Duration::from_millis(500),
"shutdown must interrupt an in-flight silent request read promptly (TOLD), \
not defer by the read's admitted 2s deadline: elapsed {:?}",
start.elapsed()
);
Ok(())
}
#[test]
fn health_idle_grows_unrelated_counters_while_accept_stays_flat()
-> Result<(), Box<dyn std::error::Error>> {
let idle = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let busy = start_health_server(loopback_ephemeral()?, SharedReadinessState::default())?;
let deadline = Instant::now() + Duration::from_secs(2);
while (idle.accept_attempts() < 1 || busy.accept_attempts() < 1)
&& Instant::now() < deadline
{
thread::sleep(Duration::from_millis(5));
}
let idle_armed = idle.accept_attempts();
assert_eq!(idle_armed, 1, "the idle listener parks exactly one accept");
let busy_before = busy.accept_attempts();
for _ in 0..5 {
let response = get(busy.local_addr(), "/health")?;
assert_status(&response, 200);
}
let deadline = Instant::now() + Duration::from_secs(2);
while busy.accept_attempts() <= busy_before && Instant::now() < deadline {
thread::sleep(Duration::from_millis(5));
}
assert!(
busy.accept_attempts() > busy_before,
"an unrelated served request grows the busy listener's accept counter"
);
assert_eq!(
idle.accept_attempts(),
idle_armed,
"the silent listener's accept counter stays flat during the workload"
);
assert_eq!(idle.shed_count(), 0, "the silent listener sheds nothing");
idle.shutdown()?;
busy.shutdown()?;
Ok(())
}
}