#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
#![cfg_attr(
not(test),
deny(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::todo,
clippy::unimplemented,
clippy::indexing_slicing,
clippy::string_slice,
clippy::arithmetic_side_effects,
)
)]
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use subtle::ConstantTimeEq as _;
use super::membership::ClusterState;
use super::{Incarnation, NodeId};
use crate::security::hmac_sha256_hex;
pub const WIRE_VERSION: u8 = 1;
pub const CURRENT_KEY_ID: u8 = 0;
pub const MAX_FRAME_BYTES: usize = 65_536;
pub const LENGTH_PREFIX_BYTES: usize = 4;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ClusterMessage {
StatePush {
state: ClusterState,
},
Leave,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Envelope {
pub v: u8,
pub key_id: u8,
pub cluster: String,
pub sender: NodeId,
pub incarnation: Incarnation,
pub seq: u64,
pub payload: String,
pub mac: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RejectReason {
Oversize,
Malformed,
Version,
KeyId,
Cluster,
Mac,
SelfOrigin,
Replay,
Payload,
}
impl RejectReason {
pub const fn label(self) -> &'static str {
match self {
Self::Oversize => "oversize",
Self::Malformed => "malformed",
Self::Version => "version",
Self::KeyId => "key_id",
Self::Cluster => "cluster",
Self::Mac => "mac",
Self::SelfOrigin => "self_origin",
Self::Replay => "replay",
Self::Payload => "payload",
}
}
pub const fn closes_connection(self) -> bool {
matches!(self, Self::Oversize)
}
pub const fn authenticated(self) -> bool {
matches!(self, Self::SelfOrigin | Self::Replay | Self::Payload)
}
}
pub fn signing_input(
v: u8,
cluster: &str,
sender: &str,
incarnation: Incarnation,
seq: u64,
payload: &[u8],
) -> Vec<u8> {
fn delimited(out: &mut Vec<u8>, bytes: &[u8]) {
let len = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
out.extend_from_slice(&len.to_be_bytes());
out.extend_from_slice(bytes);
}
let mut out = Vec::with_capacity(
65_usize
.saturating_add(cluster.len())
.saturating_add(sender.len())
.saturating_add(payload.len()),
);
delimited(&mut out, &[v]);
delimited(&mut out, cluster.as_bytes());
delimited(&mut out, sender.as_bytes());
delimited(&mut out, &incarnation.to_be_bytes());
delimited(&mut out, &seq.to_be_bytes());
delimited(&mut out, payload);
out
}
pub fn sign_envelope(
secret: &[u8],
cluster: &str,
sender: &str,
incarnation: Incarnation,
seq: u64,
message: &ClusterMessage,
) -> Option<Envelope> {
let payload = serde_json::to_string(message).ok()?;
let mac = hmac_sha256_hex(
secret,
&signing_input(
WIRE_VERSION,
cluster,
sender,
incarnation,
seq,
payload.as_bytes(),
),
);
Some(Envelope {
v: WIRE_VERSION,
key_id: CURRENT_KEY_ID,
cluster: cluster.to_owned(),
sender: sender.to_owned(),
incarnation,
seq,
payload,
mac,
})
}
pub fn encode_frame(envelope: &Envelope) -> Option<Vec<u8>> {
let body = serde_json::to_vec(envelope).ok()?;
if body.is_empty() || body.len() > MAX_FRAME_BYTES {
return None;
}
let len = u32::try_from(body.len()).ok()?;
Some(framed(len.to_be_bytes(), &body))
}
pub fn encoded_body_len(envelope: &Envelope) -> Option<usize> {
serde_json::to_vec(envelope).ok().map(|body| body.len())
}
pub fn framed(prefix: [u8; LENGTH_PREFIX_BYTES], body: &[u8]) -> Vec<u8> {
let mut frame = Vec::with_capacity(body.len().saturating_add(LENGTH_PREFIX_BYTES));
frame.extend_from_slice(&prefix);
frame.extend_from_slice(body);
frame
}
pub const fn frame_len(prefix: [u8; LENGTH_PREFIX_BYTES]) -> Option<usize> {
match u32::from_be_bytes(prefix) {
0 => None,
declared if declared as usize > MAX_FRAME_BYTES => None,
declared => Some(declared as usize),
}
}
fn frame_body(frame: &[u8]) -> Result<&[u8], RejectReason> {
let prefix: [u8; LENGTH_PREFIX_BYTES] = frame
.get(..LENGTH_PREFIX_BYTES)
.and_then(|head| <[u8; LENGTH_PREFIX_BYTES]>::try_from(head).ok())
.ok_or(RejectReason::Malformed)?;
let declared = frame_len(prefix).ok_or(RejectReason::Oversize)?;
frame
.get(LENGTH_PREFIX_BYTES..LENGTH_PREFIX_BYTES.saturating_add(declared))
.ok_or(RejectReason::Malformed)
}
pub struct FrameVerifier {
cluster: String,
local_id: NodeId,
secret: Vec<u8>,
watermarks: BTreeMap<NodeId, (Incarnation, u64)>,
rejected: u64,
}
impl std::fmt::Debug for FrameVerifier {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FrameVerifier")
.field("cluster", &self.cluster)
.field("local_id", &self.local_id)
.field("senders", &self.watermarks.len())
.field("rejected", &self.rejected)
.finish_non_exhaustive()
}
}
impl FrameVerifier {
pub fn new(cluster: impl Into<String>, local_id: impl Into<String>, secret: Vec<u8>) -> Self {
Self {
cluster: cluster.into(),
local_id: local_id.into(),
secret,
watermarks: BTreeMap::new(),
rejected: 0,
}
}
pub fn accept(&mut self, frame: &[u8]) -> Result<(Envelope, ClusterMessage), RejectReason> {
match self.verify(frame) {
Ok(accepted) => Ok(accepted),
Err(reason) => {
self.rejected = self.rejected.saturating_add(1);
Err(reason)
}
}
}
fn verify(&mut self, frame: &[u8]) -> Result<(Envelope, ClusterMessage), RejectReason> {
let body = frame_body(frame)?;
let envelope: Envelope =
serde_json::from_slice(body).map_err(|_| RejectReason::Malformed)?;
if envelope.v != WIRE_VERSION {
return Err(RejectReason::Version);
}
if envelope.key_id != CURRENT_KEY_ID {
return Err(RejectReason::KeyId);
}
if envelope.cluster != self.cluster {
return Err(RejectReason::Cluster);
}
let expected = hmac_sha256_hex(
&self.secret,
&signing_input(
envelope.v,
&envelope.cluster,
&envelope.sender,
envelope.incarnation,
envelope.seq,
envelope.payload.as_bytes(),
),
);
if !bool::from(expected.as_bytes().ct_eq(envelope.mac.as_bytes())) {
return Err(RejectReason::Mac);
}
if envelope.sender == self.local_id {
return Err(RejectReason::SelfOrigin);
}
if let Some(&(incarnation, seq)) = self.watermarks.get(&envelope.sender)
&& (envelope.incarnation < incarnation
|| (envelope.incarnation == incarnation && envelope.seq <= seq))
{
return Err(RejectReason::Replay);
}
let message: ClusterMessage =
serde_json::from_str(&envelope.payload).map_err(|_| RejectReason::Payload)?;
self.watermarks.insert(
envelope.sender.clone(),
(envelope.incarnation, envelope.seq),
);
Ok((envelope, message))
}
pub fn forget(&mut self, sender: &str) {
self.watermarks.remove(sender);
}
pub const fn rejected_total(&self) -> u64 {
self.rejected
}
#[cfg(test)]
pub fn watermark(&self, sender: &str) -> Option<(Incarnation, u64)> {
self.watermarks.get(sender).copied()
}
}
#[cfg(test)]
mod tests {
use super::{
CURRENT_KEY_ID, ClusterMessage, Envelope, FrameVerifier, LENGTH_PREFIX_BYTES,
MAX_FRAME_BYTES, RejectReason, WIRE_VERSION, encode_frame, frame_len, framed,
sign_envelope, signing_input,
};
use crate::cluster::membership::ClusterState;
use crate::security::hmac_sha256_hex;
const SECRET: &[u8] = b"cluster-test-secret-0123456789ab";
const CLUSTER: &str = "autumn";
const LOCAL: &str = "node-local";
const REMOTE: &str = "node-remote";
fn verifier() -> FrameVerifier {
FrameVerifier::new(CLUSTER, LOCAL, SECRET.to_vec())
}
fn state_push() -> ClusterMessage {
ClusterMessage::StatePush {
state: ClusterState::default(),
}
}
fn envelope_for(
secret: &[u8],
cluster: &str,
sender: &str,
incarnation: u64,
seq: u64,
message: &ClusterMessage,
) -> Option<Envelope> {
sign_envelope(secret, cluster, sender, incarnation, seq, message)
}
fn frame_for(
secret: &[u8],
cluster: &str,
sender: &str,
incarnation: u64,
seq: u64,
message: &ClusterMessage,
) -> Vec<u8> {
envelope_for(secret, cluster, sender, incarnation, seq, message)
.as_ref()
.and_then(encode_frame)
.unwrap_or_default()
}
fn frame_bytes(body: &[u8]) -> Vec<u8> {
let len = u32::try_from(body.len()).unwrap_or(u32::MAX);
framed(len.to_be_bytes(), body)
}
fn signed_raw_payload(sender: &str, incarnation: u64, seq: u64, payload: &str) -> Envelope {
let mac = hmac_sha256_hex(
SECRET,
&signing_input(
WIRE_VERSION,
CLUSTER,
sender,
incarnation,
seq,
payload.as_bytes(),
),
);
Envelope {
v: WIRE_VERSION,
key_id: CURRENT_KEY_ID,
cluster: CLUSTER.to_owned(),
sender: sender.to_owned(),
incarnation,
seq,
payload: payload.to_owned(),
mac,
}
}
#[test]
fn verifier_debug_never_prints_the_secret() {
let mut verifier = verifier();
let frame = frame_for(SECRET, CLUSTER, REMOTE, 1, 1, &ClusterMessage::Leave);
assert!(
verifier.accept(&frame).is_ok(),
"sanity: the frame verifies"
);
let rendered = format!("{verifier:?}");
assert!(
!rendered.contains("secret"),
"the secret field must not appear in Debug output at all; got {rendered}"
);
assert!(
!rendered.contains("99, 108, 117"),
"…and certainly not as the byte array a derived Debug would print \
(`clu` = 99, 108, 117); got {rendered}"
);
assert!(
rendered.contains(CLUSTER) && rendered.contains(LOCAL),
"the diagnosable fields must still be there, or redaction has cost \
the struct its usefulness; got {rendered}"
);
}
#[test]
fn wrong_envelope_version_rejected() {
let signed = envelope_for(SECRET, CLUSTER, REMOTE, 1, 1, &state_push());
assert!(signed.is_some(), "signing must produce an envelope");
let Some(mut envelope) = signed else { return };
envelope.v = WIRE_VERSION.saturating_add(1);
let frame = encode_frame(&envelope).unwrap_or_default();
let mut verifier = verifier();
assert_eq!(
verifier.accept(&frame).err(),
Some(RejectReason::Version),
"an envelope naming another wire version must be dropped as \
`version`, not accepted and not mislabelled"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
assert!(
!RejectReason::Version.closes_connection(),
"a version mismatch drops the frame and reads on"
);
assert_eq!(
verifier.watermark(REMOTE),
None,
"a refused frame must never advance the replay watermark"
);
}
#[test]
fn unknown_key_id_rejected() {
let signed = envelope_for(SECRET, CLUSTER, REMOTE, 1, 1, &ClusterMessage::Leave);
assert!(signed.is_some(), "signing must produce an envelope");
let Some(mut envelope) = signed else { return };
envelope.key_id = CURRENT_KEY_ID.saturating_add(1);
let frame = encode_frame(&envelope).unwrap_or_default();
let mut verifier = verifier();
assert_eq!(
verifier.accept(&frame).err(),
Some(RejectReason::KeyId),
"an envelope naming a key this node does not have must be dropped \
as `key_id` — and it is NOT re-signed here, proving the field is \
outside the MAC"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
assert_eq!(
verifier.watermark(REMOTE),
None,
"a refused frame must never advance the replay watermark"
);
}
#[test]
fn envelope_missing_key_id_is_malformed() {
let signed = envelope_for(SECRET, CLUSTER, REMOTE, 1, 1, &ClusterMessage::Leave);
assert!(signed.is_some(), "signing must produce an envelope");
let Some(envelope) = signed else { return };
let mut json = serde_json::to_value(&envelope).unwrap_or(serde_json::Value::Null);
assert!(
json.get("key_id").is_some(),
"sanity: a signed envelope must carry key_id in the first place"
);
if let Some(object) = json.as_object_mut() {
object.remove("key_id");
}
let body = serde_json::to_vec(&json).unwrap_or_default();
let mut verifier = verifier();
assert_eq!(
verifier.accept(&frame_bytes(&body)).err(),
Some(RejectReason::Malformed),
"an envelope with key_id omitted must be dropped as `malformed`, \
matching the normative receive path — the parser and the published \
format must agree on what is well-formed"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
}
#[test]
fn unknown_payload_variant_rejected_after_the_mac() {
let mut verifier = verifier();
let unknown = signed_raw_payload(REMOTE, 4, 9, r#"{"type":"future_variant"}"#);
let frame = encode_frame(&unknown).unwrap_or_default();
assert_eq!(
verifier.accept(&frame).err(),
Some(RejectReason::Payload),
"an unknown message type inside a VALID envelope must be dropped as \
`payload` — mapping it to `malformed` would blame the envelope for \
a payload a newer peer is entitled to send"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
assert_eq!(
verifier.watermark(REMOTE),
None,
"a payload this node dropped must not advance the watermark, or the \
sender's next (understandable) frame at the same seq is replayed away"
);
let garbage = signed_raw_payload(REMOTE, 4, 10, "not json at all");
assert_eq!(
verifier
.accept(&encode_frame(&garbage).unwrap_or_default())
.err(),
Some(RejectReason::Payload),
"a malformed authenticated payload is a `payload` drop too"
);
let good = frame_for(SECRET, CLUSTER, REMOTE, 4, 11, &ClusterMessage::Leave);
assert!(
verifier.accept(&good).is_ok(),
"…and the verifier must still accept the sender's next good frame"
);
}
#[test]
fn forgetting_a_sender_accepts_a_lower_incarnation_again() {
let mut verifier = verifier();
let high = frame_for(SECRET, CLUSTER, REMOTE, 5_000, 7, &ClusterMessage::Leave);
assert!(
verifier.accept(&high).is_ok(),
"sanity: the departing node's frame must be accepted first"
);
let rejoin = frame_for(SECRET, CLUSTER, REMOTE, 42, 0, &ClusterMessage::Leave);
assert_eq!(
verifier.accept(&rejoin).err(),
Some(RejectReason::Replay),
"sanity: while the watermark stands, a lower incarnation is a replay \
— otherwise this test proves nothing"
);
verifier.forget(REMOTE);
assert_eq!(
verifier.watermark(REMOTE),
None,
"forgetting a sender must drop its watermark row"
);
assert!(
verifier.accept(&rejoin).is_ok(),
"after the sender is forgotten its lower incarnation must be accepted \
as a fresh sender — this is the only way back for a node that \
restarted behind a backward clock step"
);
assert_eq!(
verifier.watermark(REMOTE),
Some((42, 0)),
"…and the fresh watermark must start from the frame just accepted"
);
}
#[test]
fn envelope_roundtrip() {
let message = state_push();
let frame = frame_for(SECRET, CLUSTER, REMOTE, 1, 1, &message);
assert!(
!frame.is_empty(),
"signing and encoding a state push must produce a frame"
);
let result = verifier().accept(&frame);
assert!(
matches!(&result, Ok((envelope, decoded))
if envelope.sender == REMOTE && decoded == &message),
"a correctly signed frame must round-trip to its envelope and message; got {result:?}"
);
}
#[test]
fn tampered_frame_rejected() {
let envelope = envelope_for(SECRET, CLUSTER, REMOTE, 1, 1, &state_push());
assert!(envelope.is_some(), "signing must produce an envelope");
let Some(mut envelope) = envelope else { return };
envelope.payload.push_str("-tampered");
let frame = encode_frame(&envelope).unwrap_or_default();
let mut verifier = verifier();
let result = verifier.accept(&frame);
assert_eq!(
result.err(),
Some(RejectReason::Mac),
"a flipped payload byte must fail the MAC before the payload is parsed"
);
assert_eq!(
verifier.rejected_total(),
1,
"a rejected frame must be counted (a silent drop is not observable)"
);
assert!(
!RejectReason::Mac.closes_connection(),
"a MAC failure drops the frame and reads on — it never closes the connection"
);
}
#[test]
fn wrong_secret_rejected() {
let frame = frame_for(
b"a-completely-different-secret-xx",
CLUSTER,
REMOTE,
1,
1,
&ClusterMessage::Leave,
);
let mut verifier = verifier();
let result = verifier.accept(&frame);
assert_eq!(
result.err(),
Some(RejectReason::Mac),
"a frame signed with another secret must not verify"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
}
#[test]
fn wrong_cluster_name_rejected() {
let frame = frame_for(
SECRET,
"some-other-cluster",
REMOTE,
1,
1,
&ClusterMessage::Leave,
);
let mut verifier = verifier();
let result = verifier.accept(&frame);
assert_eq!(
result.err(),
Some(RejectReason::Cluster),
"a frame naming a different cluster must be refused even under the same secret"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
}
#[test]
fn stale_sequence_dropped() {
let frame = frame_for(SECRET, CLUSTER, REMOTE, 1, 5, &ClusterMessage::Leave);
let mut verifier = verifier();
assert!(
verifier.accept(&frame).is_ok(),
"the first frame at (incarnation 1, seq 5) must be accepted"
);
assert_eq!(
verifier.watermark(REMOTE),
Some((1, 5)),
"accepting a frame must raise the per-sender watermark"
);
let replayed = verifier.accept(&frame);
assert_eq!(
replayed.err(),
Some(RejectReason::Replay),
"replaying a frame at or below the watermark must be dropped"
);
assert_eq!(verifier.rejected_total(), 1, "the replay must be counted");
let rejoined = frame_for(SECRET, CLUSTER, REMOTE, 2, 0, &ClusterMessage::Leave);
assert!(
verifier.accept(&rejoined).is_ok(),
"a higher incarnation at seq 0 must be accepted, not treated as a replay"
);
assert_eq!(
verifier.watermark(REMOTE),
Some((2, 0)),
"a higher incarnation must adopt and reset the sequence watermark"
);
let stale_incarnation = frame_for(SECRET, CLUSTER, REMOTE, 1, 6, &ClusterMessage::Leave);
assert_eq!(
verifier.accept(&stale_incarnation).err(),
Some(RejectReason::Replay),
"a frame from a lower incarnation must be dropped once a higher one is known"
);
}
#[test]
fn self_origin_frame_dropped() {
let frame = frame_for(SECRET, CLUSTER, LOCAL, 1, 1, &ClusterMessage::Leave);
let mut verifier = verifier();
let result = verifier.accept(&frame);
assert_eq!(
result.err(),
Some(RejectReason::SelfOrigin),
"a frame whose authenticated sender is this node must be dropped"
);
assert_eq!(
verifier.rejected_total(),
1,
"the rejection must be counted"
);
}
#[test]
fn oversized_frame_rejected_before_alloc() {
let cap = u32::try_from(MAX_FRAME_BYTES).unwrap_or(u32::MAX);
assert_eq!(
frame_len(cap.saturating_add(1).to_be_bytes()),
None,
"a length prefix above MAX_FRAME_BYTES must be refused before allocating"
);
assert_eq!(
frame_len(u32::MAX.to_be_bytes()),
None,
"a 4 GiB length prefix must be refused before allocating"
);
assert_eq!(
frame_len(0u32.to_be_bytes()),
None,
"a zero-length frame is malformed"
);
assert_eq!(
frame_len(64u32.to_be_bytes()),
Some(64),
"a legal length prefix must be accepted (otherwise nothing can ever be read)"
);
let mut oversized = cap.saturating_add(1).to_be_bytes().to_vec();
oversized.extend_from_slice(b"{}");
assert_eq!(
verifier().accept(&oversized).err(),
Some(RejectReason::Oversize),
"the verifier must refuse an oversized declared length"
);
assert!(
RejectReason::Oversize.closes_connection(),
"a bad length prefix desynchronizes the framing, so the connection must close"
);
}
#[test]
fn only_post_mac_verdicts_count_as_authenticated() {
for reason in [
RejectReason::Oversize,
RejectReason::Malformed,
RejectReason::Version,
RejectReason::KeyId,
RejectReason::Cluster,
RejectReason::Mac,
] {
assert!(
!reason.authenticated(),
"{} is reached at or before the MAC check, so it refuses bytes \
anyone who can reach the port could have sent",
reason.label()
);
}
for reason in [
RejectReason::SelfOrigin,
RejectReason::Replay,
RejectReason::Payload,
] {
assert!(
reason.authenticated(),
"{} is only reachable past a verified MAC, so the frame came \
from a holder of the secret",
reason.label()
);
}
}
#[test]
fn truncated_or_malformed_frame_dropped_without_panic() {
let mut verifier = verifier();
let mut truncated = 64u32.to_be_bytes().to_vec();
truncated.extend_from_slice(b"only-a-few-bytes");
let mut junk = 18u32.to_be_bytes().to_vec();
junk.extend_from_slice(b"not-a-frame-at-all");
for bad in [b"".as_slice(), b"\x00".as_slice(), &truncated, &junk] {
let result = verifier.accept(bad);
assert!(
result.is_err(),
"malformed input must be dropped, never accepted: {bad:?}"
);
assert!(
result
.err()
.is_some_and(|reason| !reason.closes_connection()),
"a malformed body drops the frame and reads on; only a bad length \
prefix closes the connection: {bad:?}"
);
}
assert_eq!(
LENGTH_PREFIX_BYTES, 4,
"the length prefix width is part of the wire contract"
);
let good = frame_for(SECRET, CLUSTER, REMOTE, 1, 1, &ClusterMessage::Leave);
assert!(
verifier.accept(&good).is_ok(),
"after malformed input the verifier must still accept a valid frame"
);
}
}