use super::node::Node;
use super::stats::NodeBasicStats;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tokio::net::{TcpListener, TcpStream};
pub const MAX_CLUSTER_MESSAGE_SIZE: usize = 1024 * 1024;
const MAX_TIMESTAMP_SKEW_MS: u64 = 5 * 60 * 1000;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ClusterMessage {
JoinRequest(Node),
JoinResponse {
success: bool,
peers: Vec<Node>,
},
Heartbeat {
from: String,
sequence: u64,
stats: Option<NodeBasicStats>,
},
Leave {
from: String,
},
Replication(crate::sync::SyncMessage),
}
#[async_trait::async_trait]
pub trait Transport: Send + Sync {
async fn send(&self, to: &str, msg: ClusterMessage) -> Result<()>;
async fn broadcast(&self, msg: ClusterMessage) -> Result<()>;
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SignedClusterMessage {
pub v: u8,
pub ts: u64,
pub nonce: String,
pub sig: String,
pub payload: String,
}
fn hmac_hex(secret: &str, ts: u64, nonce: &str, payload: &str) -> String {
use hmac::{Hmac, Mac};
use sha2::Sha256;
let mut mac =
Hmac::<Sha256>::new_from_slice(secret.as_bytes()).expect("HMAC accepts keys of any size");
mac.update(format!("{}:{}:{}", ts, nonce, payload).as_bytes());
hex::encode(mac.finalize().into_bytes())
}
pub fn seal_cluster_message(msg: &ClusterMessage, secret: Option<&str>) -> Result<Vec<u8>> {
let payload = serde_json::to_string(msg)?;
match secret {
Some(secret) if !secret.is_empty() => {
let ts = chrono::Utc::now().timestamp_millis() as u64;
let nonce = uuid::Uuid::new_v4().to_string();
let sig = hmac_hex(secret, ts, &nonce, &payload);
Ok(serde_json::to_vec(&SignedClusterMessage {
v: 1,
ts,
nonce,
sig,
payload,
})?)
}
_ => anyhow::bail!(
"refusing to send an unauthenticated cluster message: no cluster secret is \
configured. Set `cluster.keyfile` to a file containing a shared secret of at \
least 32 bytes, identical on every node."
),
}
}
pub fn open_cluster_message(data: &[u8], secret: Option<&str>) -> Result<ClusterMessage> {
match secret {
Some(secret) if !secret.is_empty() => {
let envelope: SignedClusterMessage = serde_json::from_slice(data)
.map_err(|_| anyhow::anyhow!("unsigned or malformed cluster message rejected"))?;
let now = chrono::Utc::now().timestamp_millis() as u64;
if envelope.ts.abs_diff(now) > MAX_TIMESTAMP_SKEW_MS {
anyhow::bail!("cluster message timestamp outside replay window");
}
let expected = hmac_hex(secret, envelope.ts, &envelope.nonce, &envelope.payload);
if !crate::server::auth::constant_time_eq(expected.as_bytes(), envelope.sig.as_bytes())
{
anyhow::bail!("cluster message signature mismatch");
}
Ok(serde_json::from_str(&envelope.payload)?)
}
_ => anyhow::bail!(
"refusing to accept an unauthenticated cluster message: no cluster secret is \
configured. Until one is, this node cannot take part in a cluster."
),
}
}
pub struct TcpTransport {
local_address: String,
secret: Option<String>,
}
impl TcpTransport {
pub fn new(local_address: String, secret: Option<String>) -> Self {
Self {
local_address,
secret,
}
}
pub async fn listen(&self) -> Result<TcpListener> {
let listener = TcpListener::bind(&self.local_address).await?;
Ok(listener)
}
pub async fn connect_and_send_signed(
addr: &str,
msg: ClusterMessage,
secret: Option<&str>,
) -> Result<()> {
let mut stream = TcpStream::connect(addr).await?;
let data = seal_cluster_message(&msg, secret)?;
stream.write_all(&data).await?;
Ok(())
}
pub async fn connect_and_send(addr: &str, msg: ClusterMessage) -> Result<()> {
Self::connect_and_send_signed(addr, msg, None).await
}
}
#[async_trait::async_trait]
impl Transport for TcpTransport {
async fn send(&self, to: &str, msg: ClusterMessage) -> Result<()> {
Self::connect_and_send_signed(to, msg, self.secret.as_deref()).await
}
async fn broadcast(&self, _msg: ClusterMessage) -> Result<()> {
Ok(())
}
}
#[cfg(test)]
mod cluster_auth_tests {
use super::*;
fn message() -> ClusterMessage {
ClusterMessage::Leave {
from: "node-a".to_string(),
}
}
const SECRET: &str = "a-shared-cluster-secret-at-least-32-bytes";
#[test]
fn a_signed_message_round_trips() {
let sealed = seal_cluster_message(&message(), Some(SECRET)).unwrap();
assert!(open_cluster_message(&sealed, Some(SECRET)).is_ok());
}
#[test]
fn sending_without_a_secret_is_refused_rather_than_sent_in_the_clear() {
for absent in [None, Some(""), Some(" ")] {
let result = seal_cluster_message(&message(), absent.map(str::trim));
assert!(result.is_err(), "sent unauthenticated for {absent:?}");
}
}
#[test]
fn receiving_without_a_secret_is_refused_rather_than_trusted() {
let raw = serde_json::to_vec(&message()).unwrap();
assert!(open_cluster_message(&raw, None).is_err());
assert!(open_cluster_message(&raw, Some("")).is_err());
}
#[test]
fn a_node_with_a_secret_still_rejects_an_unsigned_message() {
let raw = serde_json::to_vec(&message()).unwrap();
assert!(open_cluster_message(&raw, Some(SECRET)).is_err());
}
#[test]
fn a_message_signed_with_another_secret_is_rejected() {
let sealed =
seal_cluster_message(&message(), Some("some-other-cluster-secret-32b")).unwrap();
assert!(open_cluster_message(&sealed, Some(SECRET)).is_err());
}
#[test]
fn a_tampered_payload_is_rejected() {
let sealed = seal_cluster_message(&message(), Some(SECRET)).unwrap();
let mut envelope: serde_json::Value = serde_json::from_slice(&sealed).unwrap();
envelope["payload"] = serde_json::Value::String(
serde_json::to_string(&ClusterMessage::Leave {
from: "a-node-the-attacker-wants-evicted".into(),
})
.unwrap(),
);
let altered = serde_json::to_vec(&envelope).unwrap();
assert!(open_cluster_message(&altered, Some(SECRET)).is_err());
}
#[test]
fn the_refusal_says_what_to_configure() {
let err = seal_cluster_message(&message(), None)
.unwrap_err()
.to_string();
assert!(err.contains("cluster.keyfile"), "{err}");
assert!(err.contains("every node"), "{err}");
}
}