use alloc::vec::Vec;
use bytes::Bytes;
use core::fmt;
use crate::batch::Entry;
use crate::types::{NodeId, PersistId, ProposalId, Slot};
const MILLI: u64 = 1_000_000;
const MESSAGE_OVERHEAD: u64 = 1024;
#[derive(Clone, Debug)]
#[cfg_attr(feature = "model", derive(Hash))]
#[non_exhaustive]
pub struct Config {
pub cluster_id: [u8; 16],
pub id: NodeId,
pub members: Vec<NodeId>,
pub seed: [u8; 32],
pub max_batch_bytes: u32,
pub max_message_bytes: u32,
pub max_in_flight_slots: u32,
pub max_outstanding_proposals: u32,
pub retransmit_interval: u64,
pub hedge_min: u64,
pub hedge_max: u64,
pub leader_order: Vec<NodeId>,
}
impl Config {
pub fn new(cluster_id: [u8; 16], id: NodeId, members: Vec<NodeId>, seed: [u8; 32]) -> Self {
Self {
cluster_id,
id,
members,
seed,
max_batch_bytes: 1 << 20,
max_message_bytes: 16 << 20,
max_in_flight_slots: 8,
max_outstanding_proposals: 4096,
retransmit_interval: 50 * MILLI,
hedge_min: MILLI,
hedge_max: 1000 * MILLI,
leader_order: Vec::new(),
}
}
pub(crate) fn leaders(&self) -> Vec<NodeId> {
if self.leader_order.is_empty() {
let mut order = self.members.clone();
order.sort_unstable();
order
} else {
self.leader_order.clone()
}
}
pub(crate) fn wire_cluster_id(&self) -> [u8; 16] {
let mut id = self.cluster_id;
let mut by_id = self.members.clone();
by_id.sort_unstable();
if self.leaders() != by_id {
let fnv = self
.leader_order
.iter()
.flat_map(|n| n.0.to_le_bytes())
.fold(0xcbf2_9ce4_8422_2325_u64, |h, b| (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3));
for (b, f) in id[8..].iter_mut().zip(fnv.to_le_bytes()) {
*b ^= f;
}
}
id
}
pub(crate) fn validate(&self) -> Result<(), Error> {
let mut sorted = self.members.clone();
sorted.sort_unstable();
sorted.dedup();
let checks = [
(!self.members.is_empty(), "members must not be empty"),
(self.members.len() <= crate::schedule::MAX_MEMBERS, "at most 1024 members"),
(sorted.len() == self.members.len(), "members must be unique"),
(self.max_batch_bytes >= 64, "max_batch_bytes must be at least 64"),
(
u64::from(self.max_message_bytes) >= 2 * u64::from(self.max_batch_bytes) + MESSAGE_OVERHEAD,
"max_message_bytes must hold two full batches plus 1 KiB of framing",
),
(self.max_in_flight_slots > 0, "max_in_flight_slots must be positive"),
(
u64::from(self.max_in_flight_slots) <= crate::schedule::LAG,
"max_in_flight_slots must be at most 64",
),
(
self.hedge_max > 0 && self.hedge_min <= self.hedge_max,
"hedge bounds must satisfy 0 <= hedge_min <= hedge_max, hedge_max > 0",
),
(self.max_outstanding_proposals > 0, "max_outstanding_proposals must be positive"),
(self.retransmit_interval > 0, "retransmit_interval must be positive"),
(
self.leader_order.is_empty() || {
let mut order = self.leader_order.clone();
order.sort_unstable();
order == sorted
},
"leader_order must list every member exactly once",
),
];
match checks.iter().find(|(ok, _)| !ok) {
Some((_, why)) => Err(Error::InvalidConfig(why)),
None => Ok(()),
}
}
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum Input {
Message(Bytes),
Persisted(PersistId),
StorageFailed(PersistId),
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "model", derive(Hash))]
pub struct Outbound {
pub to: NodeId,
pub bytes: Bytes,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "model", derive(Hash))]
#[non_exhaustive]
pub enum DropReason {
Fenced,
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "model", derive(Hash))]
#[non_exhaustive]
pub enum Action {
Persist {
id: PersistId,
records: Vec<Bytes>,
then: Vec<Outbound>,
replace: bool,
},
Send(Outbound),
Commit {
slot: Slot,
entries: Vec<Entry>,
},
Committed {
id: ProposalId,
slot: Slot,
},
ReadReady {
id: ProposalId,
slot: Slot,
},
SnapshotNeeded {
peer: NodeId,
through: Slot,
},
Reconfigured {
epoch: u64,
from: Slot,
members: Vec<NodeId>,
},
Dropped {
id: ProposalId,
reason: DropReason,
},
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[cfg_attr(feature = "model", derive(Hash))]
#[non_exhaustive]
pub struct Metrics {
pub messages_rejected: u64,
pub retransmits: u64,
pub catch_up_retries: u64,
pub slots_decided: u64,
pub fast_path_decisions: u64,
pub corrupt_batches: u64,
pub forwarded: u64,
pub hedged: u64,
pub retained_slots: u64,
pub live_registers: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum Error {
InvalidConfig(&'static str),
CorruptRecord {
index: usize,
},
Busy,
PayloadTooLarge,
Fenced,
NotDrained,
UnknownPersistId,
InvalidCoreState,
ReconfigurationPending,
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Error::InvalidConfig(why) => write!(f, "invalid config: {why}"),
Error::CorruptRecord { index } => write!(f, "durable record {index} is corrupt"),
Error::Busy => f.write_str("too many outstanding proposals"),
Error::PayloadTooLarge => f.write_str("payload exceeds max_batch_bytes"),
Error::Fenced => f.write_str("replica is fenced after a storage failure"),
Error::NotDrained => f.write_str("poll_actions must drain actions before the next call"),
Error::UnknownPersistId => f.write_str("unknown persist id"),
Error::InvalidCoreState => f.write_str("invalid core state"),
Error::ReconfigurationPending => f.write_str("a membership change has not taken effect yet"),
}
}
}
impl core::error::Error for Error {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_wire_identity_is_stable_across_versions() {
let members = alloc::vec![NodeId(0), NodeId(1), NodeId(2)];
let mut cfg = Config::new([7; 16], NodeId(0), members, [1; 32]);
assert_eq!(cfg.wire_cluster_id(), [7; 16], "the default order leaves the id as configured");
cfg.leader_order = alloc::vec![NodeId(2), NodeId(0), NodeId(1)];
let mut expected = [7u8; 16];
expected[8..].copy_from_slice(&[0x61, 0x1e, 0x70, 0x7b, 0x15, 0x0b, 0xf3, 0x99]);
assert_eq!(cfg.wire_cluster_id(), expected);
}
}