use super::node::Node;
use super::stats::NodeBasicStats;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tokio::net::{TcpListener, TcpStream};
pub const MAX_CLUSTER_MESSAGE_SIZE: usize = 1024 * 1024;
const MAX_TIMESTAMP_SKEW_MS: u64 = 5 * 60 * 1000;
const MAX_REMEMBERED_NONCES: usize = 200_000;
struct NonceCache {
seen: std::collections::HashSet<String>,
order: std::collections::VecDeque<(u64, String)>,
}
impl NonceCache {
fn new() -> Self {
Self {
seen: std::collections::HashSet::new(),
order: std::collections::VecDeque::new(),
}
}
fn evict_expired(&mut self, now_ms: u64) {
let horizon = now_ms.saturating_sub(2 * MAX_TIMESTAMP_SKEW_MS);
while let Some((at, _)) = self.order.front() {
if *at >= horizon {
break;
}
if let Some((_, nonce)) = self.order.pop_front() {
self.seen.remove(&nonce);
}
}
}
fn insert(&mut self, nonce: &str, now_ms: u64) -> bool {
self.evict_expired(now_ms);
if self.seen.contains(nonce) {
return false;
}
if self.seen.len() >= MAX_REMEMBERED_NONCES {
tracing::warn!("cluster nonce cache full; rejecting signed message");
return false;
}
self.seen.insert(nonce.to_string());
self.order.push_back((now_ms, nonce.to_string()));
true
}
}
fn nonce_cache() -> &'static std::sync::Mutex<NonceCache> {
static CACHE: std::sync::OnceLock<std::sync::Mutex<NonceCache>> = std::sync::OnceLock::new();
CACHE.get_or_init(|| std::sync::Mutex::new(NonceCache::new()))
}
pub fn remember_nonce(nonce: &str) -> bool {
let now = chrono::Utc::now().timestamp_millis() as u64;
match nonce_cache().lock() {
Ok(mut cache) => cache.insert(nonce, now),
Err(poisoned) => poisoned.into_inner().insert(nonce, now),
}
}
const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ClusterMessage {
JoinRequest(Node),
JoinResponse {
success: bool,
peers: Vec<Node>,
},
Heartbeat {
from: String,
sequence: u64,
stats: Option<NodeBasicStats>,
},
Leave {
from: String,
},
Replication(crate::sync::SyncMessage),
}
#[async_trait::async_trait]
pub trait Transport: Send + Sync {
async fn send(&self, to: &str, msg: ClusterMessage) -> Result<()>;
async fn broadcast(&self, msg: ClusterMessage) -> Result<()>;
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SignedClusterMessage {
pub v: u8,
pub ts: u64,
pub nonce: String,
pub sig: String,
pub payload: String,
}
fn hmac_hex(secret: &str, ts: u64, nonce: &str, payload: &str) -> String {
use hmac::{Hmac, Mac};
use sha2::Sha256;
let mut mac =
Hmac::<Sha256>::new_from_slice(secret.as_bytes()).expect("HMAC accepts keys of any size");
mac.update(format!("{}:{}:{}", ts, nonce, payload).as_bytes());
hex::encode(mac.finalize().into_bytes())
}
pub fn seal_cluster_message(msg: &ClusterMessage, secret: Option<&str>) -> Result<Vec<u8>> {
let payload = serde_json::to_string(msg)?;
match secret {
Some(secret) if !secret.is_empty() => {
let ts = chrono::Utc::now().timestamp_millis() as u64;
let nonce = uuid::Uuid::new_v4().to_string();
let sig = hmac_hex(secret, ts, &nonce, &payload);
Ok(serde_json::to_vec(&SignedClusterMessage {
v: 1,
ts,
nonce,
sig,
payload,
})?)
}
_ => anyhow::bail!(
"refusing to send an unauthenticated cluster message: no cluster secret is \
configured. Set `cluster.keyfile` to a file containing a shared secret of at \
least 32 bytes, identical on every node."
),
}
}
pub fn open_cluster_message(data: &[u8], secret: Option<&str>) -> Result<ClusterMessage> {
match secret {
Some(secret) if !secret.is_empty() => {
let envelope: SignedClusterMessage = serde_json::from_slice(data)
.map_err(|_| anyhow::anyhow!("unsigned or malformed cluster message rejected"))?;
let now = chrono::Utc::now().timestamp_millis() as u64;
if envelope.ts.abs_diff(now) > MAX_TIMESTAMP_SKEW_MS {
anyhow::bail!("cluster message timestamp outside replay window");
}
let expected = hmac_hex(secret, envelope.ts, &envelope.nonce, &envelope.payload);
if !crate::server::auth::constant_time_eq(expected.as_bytes(), envelope.sig.as_bytes())
{
anyhow::bail!("cluster message signature mismatch");
}
if envelope.nonce.is_empty() || envelope.nonce.len() > 128 {
anyhow::bail!("cluster message nonce missing or oversized");
}
if !remember_nonce(&format!("msg:{}", envelope.nonce)) {
anyhow::bail!("cluster message replay rejected (nonce already seen)");
}
Ok(serde_json::from_str(&envelope.payload)?)
}
_ => anyhow::bail!(
"refusing to accept an unauthenticated cluster message: no cluster secret is \
configured. Until one is, this node cannot take part in a cluster."
),
}
}
pub struct TcpTransport {
local_address: String,
secret: Option<String>,
}
impl TcpTransport {
pub fn new(local_address: String, secret: Option<String>) -> Self {
Self {
local_address,
secret,
}
}
pub async fn listen(&self) -> Result<TcpListener> {
let listener = TcpListener::bind(&self.local_address).await?;
Ok(listener)
}
pub async fn connect_and_send_signed(
addr: &str,
msg: ClusterMessage,
secret: Option<&str>,
) -> Result<()> {
let data = seal_cluster_message(&msg, secret)?;
let mut stream = tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(addr))
.await
.map_err(|_| anyhow::anyhow!("connect to {} timed out", addr))??;
tokio::time::timeout(CONNECT_TIMEOUT, stream.write_all(&data))
.await
.map_err(|_| anyhow::anyhow!("write to {} timed out", addr))??;
Ok(())
}
pub async fn connect_and_send(addr: &str, msg: ClusterMessage) -> Result<()> {
Self::connect_and_send_signed(addr, msg, None).await
}
}
#[async_trait::async_trait]
impl Transport for TcpTransport {
async fn send(&self, to: &str, msg: ClusterMessage) -> Result<()> {
Self::connect_and_send_signed(to, msg, self.secret.as_deref()).await
}
async fn broadcast(&self, _msg: ClusterMessage) -> Result<()> {
Ok(())
}
}
#[cfg(test)]
mod cluster_auth_tests {
use super::*;
fn message() -> ClusterMessage {
ClusterMessage::Leave {
from: "node-a".to_string(),
}
}
const SECRET: &str = "a-shared-cluster-secret-at-least-32-bytes";
#[test]
fn a_signed_message_round_trips() {
let sealed = seal_cluster_message(&message(), Some(SECRET)).unwrap();
assert!(open_cluster_message(&sealed, Some(SECRET)).is_ok());
}
#[test]
fn sending_without_a_secret_is_refused_rather_than_sent_in_the_clear() {
for absent in [None, Some(""), Some(" ")] {
let result = seal_cluster_message(&message(), absent.map(str::trim));
assert!(result.is_err(), "sent unauthenticated for {absent:?}");
}
}
#[test]
fn receiving_without_a_secret_is_refused_rather_than_trusted() {
let raw = serde_json::to_vec(&message()).unwrap();
assert!(open_cluster_message(&raw, None).is_err());
assert!(open_cluster_message(&raw, Some("")).is_err());
}
#[test]
fn a_node_with_a_secret_still_rejects_an_unsigned_message() {
let raw = serde_json::to_vec(&message()).unwrap();
assert!(open_cluster_message(&raw, Some(SECRET)).is_err());
}
#[test]
fn a_replayed_message_is_rejected() {
let sealed = seal_cluster_message(&message(), Some(SECRET)).unwrap();
assert!(open_cluster_message(&sealed, Some(SECRET)).is_ok());
let replay = open_cluster_message(&sealed, Some(SECRET));
assert!(replay.is_err());
assert!(replay.unwrap_err().to_string().contains("replay"));
}
#[test]
fn nonce_cache_forgets_after_the_window() {
let mut cache = NonceCache::new();
let t0 = 1_000_000_000u64;
assert!(cache.insert("a", t0));
assert!(!cache.insert("a", t0 + 1));
assert!(cache.insert("a", t0 + 2 * MAX_TIMESTAMP_SKEW_MS + 1));
}
#[test]
fn a_message_signed_with_another_secret_is_rejected() {
let sealed =
seal_cluster_message(&message(), Some("some-other-cluster-secret-32b")).unwrap();
assert!(open_cluster_message(&sealed, Some(SECRET)).is_err());
}
#[test]
fn a_tampered_payload_is_rejected() {
let sealed = seal_cluster_message(&message(), Some(SECRET)).unwrap();
let mut envelope: serde_json::Value = serde_json::from_slice(&sealed).unwrap();
envelope["payload"] = serde_json::Value::String(
serde_json::to_string(&ClusterMessage::Leave {
from: "a-node-the-attacker-wants-evicted".into(),
})
.unwrap(),
);
let altered = serde_json::to_vec(&envelope).unwrap();
assert!(open_cluster_message(&altered, Some(SECRET)).is_err());
}
#[test]
fn the_refusal_says_what_to_configure() {
let err = seal_cluster_message(&message(), None)
.unwrap_err()
.to_string();
assert!(err.contains("cluster.keyfile"), "{err}");
assert!(err.contains("every node"), "{err}");
}
}