#![allow(dead_code)]
use std::collections::VecDeque;
use bytes::Bytes;
use quepaxa::{Action, Config, Input, NodeId, ProposalId, Replica, Slot, Timestamp};
pub struct Node {
pub replica: Option<Replica>,
pub disk: Vec<Bytes>,
pub applied: Vec<Bytes>,
pub next_slot: u64,
pub snapshot: Option<(u64, Vec<Bytes>, Bytes)>,
pub committed: Vec<ProposalId>,
pub ready: Vec<(ProposalId, Slot)>,
pub reconfigured: Vec<(u64, Slot, Vec<NodeId>)>,
pub retired: Option<u64>,
}
pub struct Cluster {
pub nodes: Vec<Node>,
net: VecDeque<(NodeId, Bytes)>,
pub now: u64,
configs: Vec<Config>,
}
pub fn config(id: u32, members: &[u32], start: u8) -> Config {
let members = members.iter().copied().map(NodeId).collect();
let mut seed = [0u8; 32];
seed[0] = id as u8;
seed[1] = start; Config::new(*b"example-cluster!", NodeId(id), members, seed)
}
impl Cluster {
pub fn new(n: u32) -> Self {
let members: Vec<u32> = (0..n).collect();
let configs: Vec<Config> = members.iter().map(|&i| config(i, &members, 0)).collect();
Self::from_configs(configs)
}
pub fn from_configs(configs: Vec<Config>) -> Self {
let mut cluster = Self { nodes: Vec::new(), net: VecDeque::new(), now: 0, configs: Vec::new() };
for cfg in configs {
cluster.add(cfg);
}
cluster
}
pub fn add(&mut self, cfg: Config) {
let replica = Replica::new(cfg.clone(), [], Timestamp(self.now)).expect("valid config");
self.configs.push(cfg);
self.nodes.push(Node {
replica: Some(replica),
disk: Vec::new(),
applied: Vec::new(),
next_slot: 0,
snapshot: None,
committed: Vec::new(),
ready: Vec::new(),
reconfigured: Vec::new(),
retired: None,
});
let i = self.nodes.len() - 1;
self.drain(i);
}
pub fn replica(&mut self, i: usize) -> &mut Replica {
self.nodes[i].replica.as_mut().expect("replica is up")
}
pub fn propose(&mut self, i: usize, payload: &str) -> ProposalId {
let now = Timestamp(self.now);
let id = self.replica(i).propose(now, Bytes::copy_from_slice(payload.as_bytes())).expect("accepted");
self.drain(i);
id
}
pub fn read_barrier(&mut self, i: usize) -> ProposalId {
let now = Timestamp(self.now);
let id = self.replica(i).read_barrier(now).expect("accepted");
self.drain(i);
id
}
pub fn propose_reconfiguration(&mut self, i: usize, members: &[u32], leaders: &[u32]) -> ProposalId {
let now = Timestamp(self.now);
let members = members.iter().copied().map(NodeId).collect();
let leaders = leaders.iter().copied().map(NodeId).collect();
let id = self.replica(i).propose_reconfiguration(now, members, leaders).expect("no change pending");
self.drain(i); id
}
pub fn crash(&mut self, i: usize) {
self.nodes[i].replica = None;
self.nodes[i].applied.clear();
self.nodes[i].next_slot = 0;
self.net.retain(|(to, _)| to.0 as usize != i);
}
pub fn restart(&mut self, i: usize, start: u8) {
let mut cfg = self.configs[i].clone();
cfg.seed[1] = start;
let mut replica =
Replica::new(cfg, self.nodes[i].disk.clone(), Timestamp(self.now)).expect("recovers");
let node = &mut self.nodes[i];
if let Some((through, payloads, core_state)) = node.snapshot.clone() {
node.applied = payloads;
node.next_slot = through + 1;
let mut actions = Vec::new();
replica.poll_actions(&mut actions); node.replica = Some(replica);
self.execute(i, actions);
let now = Timestamp(self.now);
let installed = self.replica(i).install_snapshot(now, Slot(through), core_state);
assert!(installed.is_ok(), "own snapshot is valid");
} else {
node.replica = Some(replica);
}
self.drain(i);
}
pub fn snapshot_and_compact(&mut self, i: usize) {
let node = &mut self.nodes[i];
let Some(through) = node.next_slot.checked_sub(1) else { return };
let core_state = node.replica.as_ref().expect("up").core_state();
node.snapshot = Some((through, node.applied.clone(), core_state.clone()));
let now = Timestamp(self.now);
self.replica(i).compact(now, Slot(through), core_state).expect("valid compaction");
self.drain(i);
}
pub fn run_for(&mut self, limit: u64) {
let end = self.now + limit;
loop {
while let Some((to, bytes)) = self.net.pop_front() {
let i = to.0 as usize;
let now = Timestamp(self.now);
if let Some(r) = self.nodes.get_mut(i).and_then(|n| n.replica.as_mut()) {
r.handle(now, Input::Message(bytes)).expect("drained before");
self.drain(i);
}
}
let next = self.nodes.iter().filter_map(|n| n.replica.as_ref()?.next_deadline()).min();
match next {
Some(Timestamp(at)) if at <= end => {
self.now = self.now.max(at);
for i in 0..self.nodes.len() {
let now = Timestamp(self.now);
let Some(r) = self.nodes[i].replica.as_mut() else { continue };
if r.next_deadline().is_some_and(|d| d <= now) {
r.handle_timeout(now).expect("drained before");
self.drain(i);
}
}
}
_ => {
self.now = end;
return;
}
}
}
}
pub fn settle(&mut self) {
self.run_for(0);
}
fn drain(&mut self, i: usize) {
let mut actions = Vec::new();
if let Some(r) = self.nodes[i].replica.as_mut() {
r.poll_actions(&mut actions);
}
self.execute(i, actions);
}
fn execute(&mut self, i: usize, actions: Vec<Action>) {
for action in actions {
match action {
Action::Persist { id, records, then, replace } => {
let disk = &mut self.nodes[i].disk;
if replace {
disk.clear();
}
disk.extend(records);
self.net.extend(then.into_iter().map(|o| (o.to, o.bytes)));
let now = Timestamp(self.now);
self.replica(i).handle(now, Input::Persisted(id)).expect("issued id");
self.drain(i);
}
Action::Send(out) => self.net.push_back((out.to, out.bytes)),
Action::Commit { slot, entries } => {
let node = &mut self.nodes[i];
if slot.0 >= node.next_slot {
node.applied.extend(entries.into_iter().map(|e| e.payload));
node.next_slot = slot.0 + 1;
}
}
Action::Committed { id, .. } => self.nodes[i].committed.push(id),
Action::ReadReady { id, slot } => self.nodes[i].ready.push((id, slot)),
Action::Reconfigured { epoch, from, members } => {
self.nodes[i].reconfigured.push((epoch, from, members))
}
Action::Retired { epoch } => self.nodes[i].retired = Some(epoch),
Action::SnapshotNeeded { peer, through } => self.install_from(i, peer.0 as usize, through),
Action::Dropped { .. } => {} _ => {} }
}
}
fn install_from(&mut self, i: usize, peer: usize, through: Slot) {
let snapshot = self.nodes[peer].snapshot.clone();
let Some((snap_through, payloads, core_state)) = snapshot.clone() else { return };
if snap_through < through.0 {
return;
}
let now = Timestamp(self.now);
let r = self.nodes[i].replica.as_mut().expect("up");
if r.install_snapshot(now, Slot(snap_through), core_state) == Ok(true) {
let node = &mut self.nodes[i];
node.applied = payloads;
node.next_slot = snap_through + 1;
node.snapshot = snapshot;
}
self.drain(i);
}
}
pub fn show(payloads: &[Bytes]) -> Vec<String> {
payloads.iter().map(|p| String::from_utf8_lossy(p).into_owned()).collect()
}