use std::collections::BTreeMap;
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use chrono::Utc;
use futures_util::{FutureExt, StreamExt};
use lapin::BasicProperties;
use lapin::options::{BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicPublishOptions};
use lapin::types::FieldTable;
use uuid::Uuid;
use crate::amqp::consumer::ReplayGuard;
use crate::amqp::reactor::protocol::{
DEFAULT_FRESHNESS_SKEW_SECS, EventRejection, ReactorEvent, ReactorReply, ReplyDecision,
queue_name, verify_event,
};
use crate::amqp::reactor::registry::{ReactorMode, event_spec, events};
use crate::error::AxiamError;
use crate::sensitive::Sensitive;
use crate::telemetry::{Outcome, Telemetry, TelemetryEvent, TelemetrySink};
const TELEMETRY_OPERATION: &str = "reactor_dispatch";
const RECONNECT_DELAY_CAP: Duration = Duration::from_secs(5);
const RECONNECT_BASE_DELAY: Duration = Duration::from_millis(200);
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReactorDecision {
Allow {
require_mfa: bool,
},
Deny {
reason: Option<String>,
},
Mutate {
patch: BTreeMap<String, String>,
},
Abstain,
}
impl ReactorDecision {
pub fn allow() -> Self {
Self::Allow { require_mfa: false }
}
pub fn require_step_up() -> Self {
Self::Allow { require_mfa: true }
}
pub fn deny(reason: impl Into<String>) -> Self {
Self::Deny {
reason: Some(reason.into()),
}
}
pub fn deny_unexplained() -> Self {
Self::Deny { reason: None }
}
pub fn mutate<K, V, I>(patch: I) -> Self
where
I: IntoIterator<Item = (K, V)>,
K: Into<String>,
V: Into<String>,
{
Self::Mutate {
patch: patch
.into_iter()
.map(|(k, v)| (k.into(), v.into()))
.collect(),
}
}
pub fn abstain() -> Self {
Self::Abstain
}
}
#[derive(Clone, Debug)]
pub struct ReactorShutdown {
tx: Arc<tokio::sync::watch::Sender<bool>>,
}
impl ReactorShutdown {
pub fn new() -> Self {
let (tx, _rx) = tokio::sync::watch::channel(false);
Self { tx: Arc::new(tx) }
}
pub fn trigger(&self) {
self.tx.send_replace(true);
}
pub fn is_triggered(&self) -> bool {
*self.tx.borrow()
}
fn subscribe(&self) -> tokio::sync::watch::Receiver<bool> {
self.tx.subscribe()
}
}
impl Default for ReactorShutdown {
fn default() -> Self {
Self::new()
}
}
pub struct ReactorConfig {
amqp_url: String,
tls: crate::amqp::transport::AmqpTlsConfig,
tenant_id: Uuid,
reactor_id: Uuid,
signing_key: Sensitive<Vec<u8>>,
mode: ReactorMode,
freshness_skew: chrono::Duration,
telemetry: Telemetry,
shutdown: Option<ReactorShutdown>,
reconnect: bool,
}
impl std::fmt::Debug for ReactorConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ReactorConfig")
.field("amqp_url", &self.amqp_url)
.field("tenant_id", &self.tenant_id)
.field("reactor_id", &self.reactor_id)
.field("signing_key", &self.signing_key)
.field("mode", &self.mode)
.field("freshness_skew_secs", &self.freshness_skew.num_seconds())
.field("reconnect", &self.reconnect)
.finish()
}
}
impl ReactorConfig {
pub fn builder() -> ReactorConfigBuilder {
ReactorConfigBuilder::default()
}
pub fn queue(&self) -> String {
queue_name(self.tenant_id, self.reactor_id)
}
}
#[derive(Default)]
pub struct ReactorConfigBuilder {
amqp_url: Option<String>,
tls: crate::amqp::transport::AmqpTlsConfig,
tenant_id: Option<Uuid>,
reactor_id: Option<Uuid>,
signing_key: Option<Sensitive<Vec<u8>>>,
mode: Option<ReactorMode>,
freshness_skew: Option<Duration>,
telemetry: Option<Arc<dyn TelemetrySink>>,
shutdown: Option<ReactorShutdown>,
reconnect: Option<bool>,
}
impl std::fmt::Debug for ReactorConfigBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ReactorConfigBuilder")
.field("amqp_url", &self.amqp_url)
.field("tenant_id", &self.tenant_id)
.field("reactor_id", &self.reactor_id)
.field("signing_key_set", &self.signing_key.is_some())
.field("mode", &self.mode)
.finish_non_exhaustive()
}
}
impl ReactorConfigBuilder {
pub fn amqp_url(mut self, url: impl Into<String>) -> Self {
self.amqp_url = Some(url.into());
self
}
pub fn tls(mut self, tls: crate::amqp::transport::AmqpTlsConfig) -> Self {
self.tls = tls;
self
}
pub fn tenant_id(mut self, tenant_id: Uuid) -> Self {
self.tenant_id = Some(tenant_id);
self
}
pub fn reactor_id(mut self, reactor_id: Uuid) -> Self {
self.reactor_id = Some(reactor_id);
self
}
pub fn signing_key(mut self, key: Sensitive<Vec<u8>>) -> Self {
self.signing_key = Some(key);
self
}
pub fn mode(mut self, mode: ReactorMode) -> Self {
self.mode = Some(mode);
self
}
pub fn freshness_skew(mut self, skew: Duration) -> Self {
self.freshness_skew = Some(skew);
self
}
pub fn telemetry_hook<S: TelemetrySink>(mut self, sink: S) -> Self {
self.telemetry = Some(Arc::new(sink));
self
}
pub fn shutdown(mut self, shutdown: ReactorShutdown) -> Self {
self.shutdown = Some(shutdown);
self
}
pub fn reconnect(mut self, reconnect: bool) -> Self {
self.reconnect = Some(reconnect);
self
}
pub fn build(self) -> Result<ReactorConfig, AxiamError> {
let amqp_url = self
.amqp_url
.ok_or_else(|| AxiamError::network("reactor config requires an amqp_url"))?;
crate::amqp::transport::ensure_amqps(&amqp_url)?;
self.tls.validate()?;
let freshness_skew = self
.freshness_skew
.and_then(|d| chrono::Duration::from_std(d).ok())
.unwrap_or_else(|| chrono::Duration::seconds(DEFAULT_FRESHNESS_SKEW_SECS));
Ok(ReactorConfig {
amqp_url,
tls: self.tls,
tenant_id: self
.tenant_id
.ok_or_else(|| AxiamError::network("reactor config requires a tenant_id"))?,
reactor_id: self
.reactor_id
.ok_or_else(|| AxiamError::network("reactor config requires a reactor_id"))?,
signing_key: self
.signing_key
.ok_or_else(|| AxiamError::network("reactor config requires a signing_key"))?,
mode: self.mode.unwrap_or(ReactorMode::Intercept),
freshness_skew,
telemetry: Telemetry::new(self.telemetry),
shutdown: self.shutdown,
reconnect: self.reconnect.unwrap_or(true),
})
}
}
pub(crate) trait ReactorDelivery {
fn data(&self) -> &[u8];
fn reply_to(&self) -> Option<&str>;
fn property_correlation_id(&self) -> Option<&str>;
fn ack(&self) -> impl Future<Output = ()> + Send;
fn nack(&self, requeue: bool) -> impl Future<Output = ()> + Send;
fn publish_reply(
&self,
reply_to: &str,
correlation_id: &str,
body: Vec<u8>,
) -> impl Future<Output = Result<(), String>> + Send;
}
struct LapinDelivery {
delivery: lapin::message::Delivery,
channel: lapin::Channel,
reply_to: Option<String>,
correlation_id: Option<String>,
}
impl ReactorDelivery for LapinDelivery {
fn data(&self) -> &[u8] {
&self.delivery.data
}
fn reply_to(&self) -> Option<&str> {
self.reply_to.as_deref()
}
fn property_correlation_id(&self) -> Option<&str> {
self.correlation_id.as_deref()
}
async fn ack(&self) {
let _ = self.delivery.acker.ack(BasicAckOptions::default()).await;
}
async fn nack(&self, requeue: bool) {
let _ = self
.delivery
.acker
.nack(BasicNackOptions {
requeue,
..Default::default()
})
.await;
}
async fn publish_reply(
&self,
reply_to: &str,
correlation_id: &str,
body: Vec<u8>,
) -> Result<(), String> {
self.channel
.basic_publish(
"".into(),
reply_to.into(),
BasicPublishOptions::default(),
&body,
BasicProperties::default()
.with_correlation_id(correlation_id.into())
.with_content_type("application/json".into()),
)
.await
.map_err(|e| format!("failed to publish reactor reply: {e}"))?
.await
.map_err(|e| format!("reactor reply was not confirmed: {e}"))?;
Ok(())
}
}
pub(crate) struct DispatchContext {
pub(crate) signing_key: Sensitive<Vec<u8>>,
pub(crate) tenant_id: Uuid,
pub(crate) mode: ReactorMode,
pub(crate) freshness_skew: chrono::Duration,
pub(crate) replay: ReplayGuard,
pub(crate) telemetry: Telemetry,
}
pub(crate) async fn dispatch_delivery<D, F, Fut>(delivery: &D, ctx: &DispatchContext, handler: &F)
where
D: ReactorDelivery,
F: Fn(ReactorEvent) -> Fut + Send + Sync,
Fut: Future<Output = ReactorDecision> + Send,
{
let Some(body) = parse_body(delivery.data()) else {
tracing::warn!(
target: "axiam_sdk::security",
"reactor delivery body failed JSON parse; nacking without requeue"
);
delivery.nack(false).await;
return;
};
let event = match verify_event(
&body,
ctx.signing_key.expose(),
Utc::now(),
ctx.freshness_skew,
) {
Ok(event) => event,
Err(rejection) => {
reject(delivery, &rejection.to_string()).await;
return;
}
};
if !ctx.replay.check_and_record_nonce(&event.nonce.to_string()) {
reject(delivery, &EventRejection::ReplayedNonce.to_string()).await;
return;
}
if event.tenant_id != ctx.tenant_id {
reject(
delivery,
"event names a different tenant than this reactor is configured for",
)
.await;
return;
}
let started = Instant::now();
let deadline = started + event.timeout();
let event_label = static_event_label(&event.event);
ctx.telemetry.emit(TelemetryEvent::RequestStart {
operation: TELEMETRY_OPERATION,
method: "AMQP",
path_template: event_label,
attempt: 1,
});
let outcome = tokio::time::timeout(
event.timeout(),
AssertUnwindSafe(handler(event.clone())).catch_unwind(),
)
.await;
let decision = match outcome {
Ok(Ok(decision)) => Some(decision),
Ok(Err(_panic)) => {
tracing::error!(
target: "axiam_sdk::security",
event = %event.event,
correlation_id = %event.correlation_id,
"reactor handler panicked; publishing no reply so the registration's failure_policy decides"
);
None
}
Err(_elapsed) => {
tracing::warn!(
target: "axiam_sdk::security",
event = %event.event,
correlation_id = %event.correlation_id,
timeout_ms = event.timeout_ms,
"reactor handler exceeded the event's timeout_ms; publishing no reply"
);
None
}
};
ctx.telemetry.emit(TelemetryEvent::RequestEnd {
operation: TELEMETRY_OPERATION,
method: "AMQP",
path_template: event_label,
attempt: 1,
status: None,
duration: started.elapsed(),
outcome: if decision.is_some() {
Outcome::Success
} else {
Outcome::Failure
},
});
if ctx.mode == ReactorMode::Listen {
delivery.ack().await;
return;
}
if let Some(decision) = decision {
publish_answer(delivery, ctx, &event, decision, deadline).await;
}
delivery.ack().await;
}
async fn publish_answer<D: ReactorDelivery>(
delivery: &D,
ctx: &DispatchContext,
event: &ReactorEvent,
decision: ReactorDecision,
deadline: Instant,
) {
let (wire_decision, reason, patch, require_mfa) = match decision {
ReactorDecision::Abstain => {
tracing::debug!(
target: "axiam_sdk::reactor",
event = %event.event,
correlation_id = %event.correlation_id,
"handler abstained; the registration's failure_policy decides"
);
return;
}
ReactorDecision::Allow { require_mfa } => (ReplyDecision::Allow, None, None, require_mfa),
ReactorDecision::Deny { reason } => (ReplyDecision::Deny, reason, None, false),
ReactorDecision::Mutate { patch } => {
if patch.is_empty() {
tracing::error!(
target: "axiam_sdk::reactor",
event = %event.event,
correlation_id = %event.correlation_id,
"handler returned a mutation with an empty patch (malformed_mutation); publishing no reply"
);
return;
}
(ReplyDecision::Mutate, None, Some(patch), false)
}
};
if require_mfa && event.event != events::LOGIN_POST_AUTH {
tracing::error!(
target: "axiam_sdk::reactor",
event = %event.event,
correlation_id = %event.correlation_id,
"require_mfa is only valid on login.post_auth (require_mfa_not_supported); publishing no reply"
);
return;
}
let mut reply = ReactorReply::answering(event, wire_decision, Utc::now());
reply.reason = reason;
reply.patch = patch;
reply.require_mfa = require_mfa;
if reply.sign(ctx.signing_key.expose()).is_err() {
tracing::error!(
target: "axiam_sdk::reactor",
correlation_id = %event.correlation_id,
"reactor reply could not be serialized for signing; publishing no reply"
);
return;
}
let Ok(body) = serde_json::to_vec(&reply) else {
tracing::error!(
target: "axiam_sdk::reactor",
correlation_id = %event.correlation_id,
"reactor reply could not be serialized; publishing no reply"
);
return;
};
if Instant::now() >= deadline {
tracing::warn!(
target: "axiam_sdk::reactor",
event = %event.event,
correlation_id = %event.correlation_id,
timeout_ms = event.timeout_ms,
"the event's window closed before the reply was ready; not publishing"
);
return;
}
let Some(reply_to) = delivery.reply_to() else {
tracing::warn!(
target: "axiam_sdk::reactor",
correlation_id = %event.correlation_id,
"delivery carried no reply_to property; nowhere to publish the reply"
);
return;
};
let property_correlation = delivery
.property_correlation_id()
.map(str::to_owned)
.unwrap_or_else(|| event.correlation_id.to_string());
if let Err(message) = delivery
.publish_reply(reply_to, &property_correlation, body)
.await
{
tracing::warn!(
target: "axiam_sdk::reactor",
correlation_id = %event.correlation_id,
error = %message,
"reactor reply could not be published"
);
}
}
fn parse_body(data: &[u8]) -> Option<serde_json::Value> {
serde_json::from_slice(data).ok()
}
async fn reject<D: ReactorDelivery>(delivery: &D, reason: &str) {
tracing::warn!(
target: "axiam_sdk::security",
reason,
"reactor event rejected; nacking without requeue"
);
delivery.nack(false).await;
}
fn static_event_label(name: &str) -> &'static str {
event_spec(name).map_or("unknown_event", |spec| spec.name)
}
pub async fn reactor_serve<F, Fut>(config: ReactorConfig, handler: F) -> Result<(), AxiamError>
where
F: Fn(ReactorEvent) -> Fut + Send + Sync + 'static,
Fut: Future<Output = ReactorDecision> + Send,
{
let queue = config.queue();
let ctx = DispatchContext {
signing_key: config.signing_key,
tenant_id: config.tenant_id,
mode: config.mode,
freshness_skew: config.freshness_skew,
replay: ReplayGuard::new(config.freshness_skew),
telemetry: config.telemetry,
};
let mut shutdown_rx = config.shutdown.as_ref().map(ReactorShutdown::subscribe);
let mut consecutive_failures: u32 = 0;
loop {
if shutdown_triggered(shutdown_rx.as_ref()) {
return Ok(());
}
match serve_once(
&config.amqp_url,
&config.tls,
&queue,
&ctx,
&handler,
shutdown_rx.as_mut(),
)
.await
{
Ok(ServeExit::ShutdownRequested) => return Ok(()),
Ok(ServeExit::StreamEnded) | Err(_) if !config.reconnect => {
return Ok(());
}
Ok(ServeExit::StreamEnded) => {
consecutive_failures = consecutive_failures.saturating_add(1);
}
Err(error) => {
tracing::warn!(
target: "axiam_sdk::reactor",
queue = %queue,
error = %error,
"reactor broker connection failed; will reconnect"
);
consecutive_failures = consecutive_failures.saturating_add(1);
}
}
let delay = reconnect_delay(consecutive_failures);
if let Some(rx) = shutdown_rx.as_mut() {
tokio::select! {
_ = tokio::time::sleep(delay) => {}
_ = rx.changed() => return Ok(()),
}
} else {
tokio::time::sleep(delay).await;
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ServeExit {
ShutdownRequested,
StreamEnded,
}
pub(crate) async fn consume_loop<D, S, E, F, Fut>(
stream: &mut S,
ctx: &DispatchContext,
handler: &F,
mut shutdown_rx: Option<&mut tokio::sync::watch::Receiver<bool>>,
) -> ServeExit
where
D: ReactorDelivery,
E: std::fmt::Display,
S: futures_util::Stream<Item = Result<D, E>> + Unpin,
F: Fn(ReactorEvent) -> Fut + Send + Sync,
Fut: Future<Output = ReactorDecision> + Send,
{
loop {
if shutdown_rx.as_ref().is_some_and(|rx| *rx.borrow()) {
return ServeExit::ShutdownRequested;
}
let next = match shutdown_rx.as_mut() {
Some(rx) => {
tokio::select! {
biased;
_ = rx.changed() => return ServeExit::ShutdownRequested,
next = stream.next() => next,
}
}
None => stream.next().await,
};
let Some(item) = next else {
return ServeExit::StreamEnded;
};
match item {
Ok(delivery) => dispatch_delivery(&delivery, ctx, handler).await,
Err(e) => {
tracing::warn!(target: "axiam_sdk::reactor", error = %e, "reactor consumer stream error");
return ServeExit::StreamEnded;
}
}
}
}
async fn serve_once<F, Fut>(
amqp_url: &str,
tls: &crate::amqp::transport::AmqpTlsConfig,
queue: &str,
ctx: &DispatchContext,
handler: &F,
shutdown_rx: Option<&mut tokio::sync::watch::Receiver<bool>>,
) -> Result<ServeExit, AxiamError>
where
F: Fn(ReactorEvent) -> Fut + Send + Sync,
Fut: Future<Output = ReactorDecision> + Send,
{
let connection = crate::amqp::transport::connect_amqps(amqp_url, tls).await?;
let channel = connection
.create_channel()
.await
.map_err(|e| AxiamError::network(format!("failed to create AMQP channel: {e}")))?;
let consumer = channel
.basic_consume(
queue.into(),
"axiam-sdk-reactor".into(),
BasicConsumeOptions::default(),
FieldTable::default(),
)
.await
.map_err(|e| AxiamError::network(format!("failed to start reactor consumer: {e}")))?;
let reply_channel = channel.clone();
let mut stream = consumer.map(move |result| {
result.map(|delivery| {
let reply_to = delivery
.properties
.reply_to()
.as_ref()
.map(|s| s.to_string());
let correlation_id = delivery
.properties
.correlation_id()
.as_ref()
.map(|s| s.to_string());
LapinDelivery {
delivery,
channel: reply_channel.clone(),
reply_to,
correlation_id,
}
})
});
Ok(consume_loop(&mut stream, ctx, handler, shutdown_rx).await)
}
fn shutdown_triggered(rx: Option<&tokio::sync::watch::Receiver<bool>>) -> bool {
rx.is_some_and(|rx| *rx.borrow())
}
fn reconnect_delay(consecutive_failures: u32) -> Duration {
let exponent = consecutive_failures.saturating_sub(1).min(16);
let backoff = RECONNECT_BASE_DELAY
.saturating_mul(1u32 << exponent)
.min(RECONNECT_DELAY_CAP);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| u64::from(d.subsec_nanos()));
let millis = backoff.as_millis().max(1) as u64;
Duration::from_millis(nanos % millis)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::amqp::reactor::protocol::ReactorEvent;
use crate::amqp::test_log::{captured_log, init_recording_subscriber};
fn fixture() -> serde_json::Value {
serde_json::from_str(include_str!(
"../../../testdata/reactor_v2_reference_vectors.json"
))
.expect("reactor reference vectors parse")
}
fn subkey() -> Vec<u8> {
let hex = fixture()["hkdf"]["derived_subkey_hex"]
.as_str()
.expect("derived_subkey_hex")
.to_owned();
(0..hex.len())
.step_by(2)
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap())
.collect()
}
fn fixture_tenant() -> Uuid {
fixture()["tenant_id"].as_str().unwrap().parse().unwrap()
}
fn signed_event_bytes(
key: &[u8],
tenant_id: Uuid,
event: &str,
timeout_ms: u32,
issued_at: chrono::DateTime<Utc>,
key_version: u8,
) -> Vec<u8> {
let mut message = ReactorEvent {
tenant_id,
event: event.to_owned(),
correlation_id: Uuid::new_v4(),
payload: serde_json::json!({"sub": "alice"}),
timeout_ms,
key_version,
nonce: Uuid::new_v4(),
issued_at,
hmac_signature: None,
};
let canonical = serde_json::to_vec(&message).unwrap();
message.hmac_signature = Some(crate::amqp::hmac::sign_payload(key, &canonical));
serde_json::to_vec(&message).unwrap()
}
fn fresh_event_bytes(key: &[u8], event: &str, timeout_ms: u32) -> Vec<u8> {
signed_event_bytes(key, fixture_tenant(), event, timeout_ms, Utc::now(), 2)
}
type PublishedReply = (String, String, Vec<u8>);
#[derive(Clone)]
struct FakeDelivery {
data: Vec<u8>,
reply_to: Option<String>,
correlation_id: Option<String>,
published: Arc<Mutex<Vec<PublishedReply>>>,
acked: Arc<AtomicUsize>,
nacked_no_requeue: Arc<AtomicUsize>,
nacked_requeue: Arc<AtomicUsize>,
}
impl FakeDelivery {
fn new(data: Vec<u8>) -> Self {
Self {
data,
reply_to: Some("amq.rabbitmq.reply-to.abc".to_owned()),
correlation_id: Some("property-correlation".to_owned()),
published: Arc::new(Mutex::new(Vec::new())),
acked: Arc::new(AtomicUsize::new(0)),
nacked_no_requeue: Arc::new(AtomicUsize::new(0)),
nacked_requeue: Arc::new(AtomicUsize::new(0)),
}
}
fn without_reply_to(mut self) -> Self {
self.reply_to = None;
self
}
fn published(&self) -> Vec<PublishedReply> {
self.published.lock().unwrap().clone()
}
fn published_reply(&self) -> ReactorReply {
let all = self.published();
assert_eq!(all.len(), 1, "expected exactly one published reply");
serde_json::from_slice(&all[0].2).expect("the reply is JSON")
}
}
impl ReactorDelivery for FakeDelivery {
fn data(&self) -> &[u8] {
&self.data
}
fn reply_to(&self) -> Option<&str> {
self.reply_to.as_deref()
}
fn property_correlation_id(&self) -> Option<&str> {
self.correlation_id.as_deref()
}
async fn ack(&self) {
self.acked.fetch_add(1, Ordering::SeqCst);
}
async fn nack(&self, requeue: bool) {
if requeue {
self.nacked_requeue.fetch_add(1, Ordering::SeqCst);
} else {
self.nacked_no_requeue.fetch_add(1, Ordering::SeqCst);
}
}
async fn publish_reply(
&self,
reply_to: &str,
correlation_id: &str,
body: Vec<u8>,
) -> Result<(), String> {
self.published.lock().unwrap().push((
reply_to.to_owned(),
correlation_id.to_owned(),
body,
));
Ok(())
}
}
fn context(key: Vec<u8>, mode: ReactorMode) -> DispatchContext {
let skew = chrono::Duration::seconds(DEFAULT_FRESHNESS_SKEW_SECS);
DispatchContext {
signing_key: Sensitive::new(key),
tenant_id: fixture_tenant(),
mode,
freshness_skew: skew,
replay: ReplayGuard::new(skew),
telemetry: Telemetry::default(),
}
}
fn intercept_context() -> DispatchContext {
context(subkey(), ReactorMode::Intercept)
}
fn counting_handler(
calls: Arc<AtomicUsize>,
decision: ReactorDecision,
) -> impl Fn(ReactorEvent) -> std::pin::Pin<Box<dyn Future<Output = ReactorDecision> + Send>>
+ Send
+ Sync {
move |_event| {
let calls = Arc::clone(&calls);
let decision = decision.clone();
Box::pin(async move {
calls.fetch_add(1, Ordering::SeqCst);
decision
})
}
}
#[tokio::test]
async fn a_verified_event_reaches_the_handler_and_its_reply_is_signed() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
let calls = Arc::new(AtomicUsize::new(0));
dispatch_delivery(
&delivery,
&intercept_context(),
&counting_handler(Arc::clone(&calls), ReactorDecision::allow()),
)
.await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(delivery.acked.load(Ordering::SeqCst), 1);
assert_eq!(delivery.nacked_no_requeue.load(Ordering::SeqCst), 0);
let reply = delivery.published_reply();
assert_eq!(reply.decision, ReplyDecision::Allow);
assert!(reply.signature_valid(&key), "the reply must be signed");
assert_eq!(reply.key_version, 2);
assert!(!reply.require_mfa);
let event: ReactorEvent = serde_json::from_slice(&delivery.data).unwrap();
assert_eq!(reply.correlation_id, event.correlation_id);
assert_eq!(reply.tenant_id, event.tenant_id);
assert_eq!(reply.event, event.event);
assert_eq!(delivery.published()[0].0, "amq.rabbitmq.reply-to.abc");
assert_eq!(delivery.published()[0].1, "property-correlation");
}
#[tokio::test]
async fn a_handler_that_panics_publishes_nothing_rather_than_an_allow() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
panic!("handler blew up");
})
.await;
assert!(
delivery.published().is_empty(),
"a crashed handler must not have an `allow` synthesized on its behalf"
);
assert_eq!(delivery.acked.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn a_handler_that_outruns_timeout_ms_publishes_nothing() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 1));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
tokio::time::sleep(Duration::from_millis(80)).await;
ReactorDecision::allow()
})
.await;
assert!(
delivery.published().is_empty(),
"a late reply is discarded anyway; publishing it spends the network for nothing"
);
}
#[tokio::test]
async fn an_abstaining_handler_publishes_nothing() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::GRANT_PRE_ASSIGN, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::abstain()
})
.await;
assert!(delivery.published().is_empty());
assert_eq!(delivery.acked.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn an_empty_mutation_publishes_nothing() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::TOKEN_PRE_ISSUE, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::Mutate {
patch: BTreeMap::new(),
}
})
.await;
assert!(delivery.published().is_empty(), "malformed_mutation");
}
#[tokio::test]
async fn a_forbidden_patch_key_is_published_unfiltered() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::TOKEN_PRE_ISSUE, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::mutate([("ext.department", "eng"), ("sub", "root")])
})
.await;
let reply = delivery.published_reply();
assert_eq!(reply.decision, ReplyDecision::Mutate);
assert!(reply.signature_valid(&key));
let patch = reply.patch.expect("a patch");
assert_eq!(patch.get("sub").map(String::as_str), Some("root"));
assert_eq!(patch.get("ext.department").map(String::as_str), Some("eng"));
}
#[tokio::test]
async fn require_mfa_rides_on_allow_for_login_post_auth() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::require_step_up()
})
.await;
let reply = delivery.published_reply();
assert_eq!(reply.decision, ReplyDecision::Allow);
assert!(reply.require_mfa);
assert!(reply.signature_valid(&key));
}
#[tokio::test]
async fn require_mfa_on_any_other_event_is_refused_client_side() {
let key = subkey();
for event in [
events::TOKEN_PRE_ISSUE,
events::USER_PRE_CREATE,
events::GRANT_PRE_ASSIGN,
] {
let delivery = FakeDelivery::new(fresh_event_bytes(&key, event, 5_000));
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::require_step_up()
})
.await;
assert!(
delivery.published().is_empty(),
"{event}: require_mfa_not_supported"
);
}
}
#[tokio::test]
async fn a_listener_never_publishes_a_reply() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
let calls = Arc::new(AtomicUsize::new(0));
dispatch_delivery(
&delivery,
&context(key, ReactorMode::Listen),
&counting_handler(Arc::clone(&calls), ReactorDecision::deny("nope")),
)
.await;
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"the handler still observes"
);
assert!(delivery.published().is_empty());
assert_eq!(delivery.acked.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn a_bad_signature_never_reaches_the_handler() {
let key = subkey();
let mut body: serde_json::Value =
serde_json::from_slice(&fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000))
.unwrap();
body["hmac_signature"] = serde_json::Value::String("00".repeat(32));
let delivery = FakeDelivery::new(serde_json::to_vec(&body).unwrap());
let calls = Arc::new(AtomicUsize::new(0));
dispatch_delivery(
&delivery,
&intercept_context(),
&counting_handler(Arc::clone(&calls), ReactorDecision::allow()),
)
.await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(delivery.published().is_empty());
assert_eq!(delivery.nacked_no_requeue.load(Ordering::SeqCst), 1);
assert_eq!(delivery.nacked_requeue.load(Ordering::SeqCst), 0);
assert_eq!(delivery.acked.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn a_v1_event_is_refused_and_a_stale_one_too() {
let key = subkey();
let calls = Arc::new(AtomicUsize::new(0));
let v1 = FakeDelivery::new(signed_event_bytes(
&key,
fixture_tenant(),
events::LOGIN_POST_AUTH,
5_000,
Utc::now(),
1,
));
dispatch_delivery(
&v1,
&intercept_context(),
&counting_handler(Arc::clone(&calls), ReactorDecision::allow()),
)
.await;
assert_eq!(v1.nacked_no_requeue.load(Ordering::SeqCst), 1);
for offset in [
chrono::Duration::seconds(DEFAULT_FRESHNESS_SKEW_SECS + 5),
-chrono::Duration::seconds(DEFAULT_FRESHNESS_SKEW_SECS + 5),
] {
let stale = FakeDelivery::new(signed_event_bytes(
&key,
fixture_tenant(),
events::LOGIN_POST_AUTH,
5_000,
Utc::now() - offset,
2,
));
dispatch_delivery(
&stale,
&intercept_context(),
&counting_handler(Arc::clone(&calls), ReactorDecision::allow()),
)
.await;
assert_eq!(stale.nacked_no_requeue.load(Ordering::SeqCst), 1);
}
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"no unverified event may reach user code"
);
}
#[tokio::test]
async fn a_replayed_nonce_is_refused_the_second_time() {
let key = subkey();
let bytes = fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000);
let ctx = intercept_context();
let calls = Arc::new(AtomicUsize::new(0));
let handler = counting_handler(Arc::clone(&calls), ReactorDecision::allow());
let first = FakeDelivery::new(bytes.clone());
dispatch_delivery(&first, &ctx, &handler).await;
let second = FakeDelivery::new(bytes);
dispatch_delivery(&second, &ctx, &handler).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(second.nacked_no_requeue.load(Ordering::SeqCst), 1);
assert!(second.published().is_empty());
}
#[tokio::test]
async fn an_event_naming_another_tenant_is_refused() {
let key = subkey();
let delivery = FakeDelivery::new(signed_event_bytes(
&key,
Uuid::new_v4(),
events::LOGIN_POST_AUTH,
5_000,
Utc::now(),
2,
));
let calls = Arc::new(AtomicUsize::new(0));
dispatch_delivery(
&delivery,
&intercept_context(),
&counting_handler(Arc::clone(&calls), ReactorDecision::allow()),
)
.await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(delivery.published().is_empty());
assert_eq!(delivery.nacked_no_requeue.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn a_delivery_with_no_reply_to_publishes_nothing() {
let key = subkey();
let delivery = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000))
.without_reply_to();
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::allow()
})
.await;
assert!(delivery.published().is_empty());
}
#[tokio::test]
async fn an_unparseable_body_is_nacked_without_requeue() {
let delivery = FakeDelivery::new(b"{not json".to_vec());
dispatch_delivery(&delivery, &intercept_context(), &|_event| async {
ReactorDecision::allow()
})
.await;
assert_eq!(delivery.nacked_no_requeue.load(Ordering::SeqCst), 1);
assert!(delivery.published().is_empty());
}
#[test]
fn the_reactor_runtime_declares_no_exchange_queue_or_binding() {
let forbidden: Vec<String> = [
"queue_declare",
"exchange_declare",
"queue_bind",
"exchange_bind",
]
.iter()
.map(|op| format!(".{op}("))
.collect();
for (name, source) in [
("runtime.rs", include_str!("runtime.rs")),
("protocol.rs", include_str!("protocol.rs")),
("registry.rs", include_str!("registry.rs")),
("mod.rs", include_str!("mod.rs")),
] {
for call in &forbidden {
assert!(
!source.contains(call.as_str()),
"{name} must not call {call} — §22.1: actors consume, they never declare topology"
);
}
}
}
#[tokio::test]
async fn shutdown_drains_the_in_flight_event_and_stops_before_the_next() {
let key = subkey();
let first = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
let second = FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000));
let shutdown = ReactorShutdown::new();
let mut rx = shutdown.subscribe();
let signal = shutdown.clone();
let mut stream = futures_util::stream::iter(vec![
Ok::<_, String>(first.clone()),
Ok::<_, String>(second.clone()),
]);
let exit = consume_loop(
&mut stream,
&intercept_context(),
&move |_event: ReactorEvent| {
let signal = signal.clone();
async move {
signal.trigger();
ReactorDecision::allow()
}
},
Some(&mut rx),
)
.await;
assert_eq!(exit, ServeExit::ShutdownRequested);
assert_eq!(
first.published().len(),
1,
"the in-flight event must be drained, not truncated"
);
assert_eq!(first.acked.load(Ordering::SeqCst), 1);
assert!(
second.published().is_empty(),
"no NEW delivery may be taken after the signal"
);
assert_eq!(second.acked.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn the_loop_reports_a_stream_that_ends_on_its_own() {
let mut stream = futures_util::stream::iter(Vec::<Result<FakeDelivery, String>>::new());
let exit = consume_loop(
&mut stream,
&intercept_context(),
&|_event: ReactorEvent| async { ReactorDecision::allow() },
None,
)
.await;
assert_eq!(exit, ServeExit::StreamEnded);
}
#[tokio::test]
async fn the_signing_key_never_appears_in_a_log_line() {
init_recording_subscriber();
let key = subkey();
let key_hex: String = key.iter().map(|b| format!("{b:02x}")).collect();
let key_debug = format!("{key:?}");
let mut tampered: serde_json::Value =
serde_json::from_slice(&fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000))
.unwrap();
tampered["hmac_signature"] = serde_json::Value::String("11".repeat(32));
let ctx = intercept_context();
dispatch_delivery(
&FakeDelivery::new(serde_json::to_vec(&tampered).unwrap()),
&ctx,
&|_event| async { ReactorDecision::allow() },
)
.await;
let replayed = fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000);
dispatch_delivery(
&FakeDelivery::new(replayed.clone()),
&ctx,
&|_event| async { ReactorDecision::allow() },
)
.await;
dispatch_delivery(&FakeDelivery::new(replayed), &ctx, &|_event| async {
ReactorDecision::allow()
})
.await;
dispatch_delivery(
&FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000)),
&ctx,
&|_event| async { panic!("boom") },
)
.await;
dispatch_delivery(
&FakeDelivery::new(fresh_event_bytes(&key, events::TOKEN_PRE_ISSUE, 5_000)),
&ctx,
&|_event| async { ReactorDecision::require_step_up() },
)
.await;
dispatch_delivery(
&FakeDelivery::new(fresh_event_bytes(&key, events::LOGIN_POST_AUTH, 5_000))
.without_reply_to(),
&ctx,
&|_event| async { ReactorDecision::allow() },
)
.await;
let log = captured_log();
assert!(!log.is_empty(), "these paths must log something");
assert!(
!log.contains(&key_hex),
"the tenant signing key must never be logged (§22.12): {log}"
);
assert!(
!log.contains(&key_debug),
"the tenant signing key must never be logged (§22.12): {log}"
);
assert!(!log.contains("super-secret"));
}
#[test]
fn the_reconnect_delay_never_exceeds_the_cap() {
for failures in 0..40 {
assert!(reconnect_delay(failures) <= RECONNECT_DELAY_CAP);
}
}
#[test]
fn a_shutdown_signal_is_idempotent() {
let shutdown = ReactorShutdown::new();
assert!(!shutdown.is_triggered());
shutdown.trigger();
shutdown.trigger();
assert!(shutdown.is_triggered());
}
#[test]
fn the_config_debug_output_never_renders_the_signing_key() {
let config = ReactorConfig::builder()
.amqp_url("amqps://broker.example.com:5671")
.tenant_id(Uuid::nil())
.reactor_id(Uuid::nil())
.signing_key(Sensitive::new(b"super-secret-subkey".to_vec()))
.build()
.expect("valid config");
let rendered = format!("{config:?}");
assert!(!rendered.contains("super-secret-subkey"));
assert!(!rendered.contains("115, 117, 112")); }
#[test]
fn a_plaintext_broker_url_is_refused() {
let error = ReactorConfig::builder()
.amqp_url("amqp://broker.example.com:5672")
.tenant_id(Uuid::nil())
.reactor_id(Uuid::nil())
.signing_key(Sensitive::new(vec![1, 2, 3]))
.build()
.expect_err("plaintext must be refused");
assert!(matches!(error, AxiamError::Network { .. }));
}
#[test]
fn the_queue_name_is_derived_for_this_reactor_only() {
let tenant: Uuid = "11111111-1111-1111-1111-111111111111".parse().unwrap();
let reactor: Uuid = "99999999-9999-9999-9999-999999999999".parse().unwrap();
let config = ReactorConfig::builder()
.amqp_url("amqps://broker.example.com:5671")
.tenant_id(tenant)
.reactor_id(reactor)
.signing_key(Sensitive::new(vec![0u8; 32]))
.build()
.unwrap();
assert_eq!(
config.queue(),
"axiam.reactor.q.11111111-1111-1111-1111-111111111111.99999999-9999-9999-9999-999999999999"
);
}
#[test]
fn a_missing_required_field_is_a_build_error() {
assert!(ReactorConfig::builder().build().is_err());
assert!(
ReactorConfig::builder()
.amqp_url("amqps://b.example.com")
.build()
.is_err()
);
}
#[test]
fn telemetry_labels_come_from_the_registry_not_the_wire() {
assert_eq!(static_event_label("token.pre_issue"), "token.pre_issue");
assert_eq!(static_event_label("../../etc/passwd"), "unknown_event");
assert_eq!(static_event_label("authz.check"), "unknown_event");
}
#[test]
fn decision_constructors_produce_the_documented_shapes() {
assert_eq!(
ReactorDecision::allow(),
ReactorDecision::Allow { require_mfa: false }
);
assert_eq!(
ReactorDecision::require_step_up(),
ReactorDecision::Allow { require_mfa: true }
);
assert_eq!(
ReactorDecision::deny("nope"),
ReactorDecision::Deny {
reason: Some("nope".into())
}
);
assert_eq!(
ReactorDecision::deny_unexplained(),
ReactorDecision::Deny { reason: None }
);
assert_eq!(
ReactorDecision::mutate([("ext.a", "b")]),
ReactorDecision::Mutate {
patch: BTreeMap::from([("ext.a".to_string(), "b".to_string())])
}
);
assert_eq!(ReactorDecision::abstain(), ReactorDecision::Abstain);
}
}