use std::any::Any;
use std::collections::BTreeMap;
use std::marker::PhantomData;
use serde::Serialize;
use serde::de::DeserializeOwned;
use crate::cell::Source;
use crate::context::Context;
use crate::crdt::{
CellCrdt, CrdtPlane, HlcStamp, LwwRegister, OpLog, ReplicatedCell, StampFrontier,
};
use crate::distributed::{NodeId, PeerId};
use crate::ipc::{CrdtOp, CrdtSync, DeltaSinceRequest, IpcValue, KeyIndex, NodeKey, WireStamp};
const FAMILY_NODE_BASE: u64 = 1 << 48;
trait FamilyFactory {
fn namespace(&self) -> &str;
fn materialize(&self, ctx: &Context, bytes: &[u8]) -> Option<Box<dyn PlaneCell>>;
}
struct LwwFamilyFactory<V> {
namespace: String,
_marker: PhantomData<fn() -> V>,
}
impl<V> FamilyFactory for LwwFamilyFactory<V>
where
V: Clone + PartialEq + Serialize + DeserializeOwned + 'static,
{
fn namespace(&self) -> &str {
&self.namespace
}
fn materialize(&self, ctx: &Context, bytes: &[u8]) -> Option<Box<dyn PlaneCell>> {
let remote: LwwRegister<V> = serde_json::from_slice(bytes).ok()?;
Some(Box::new(ReplicatedCell::bind(ctx, remote)))
}
}
trait PlaneCell {
fn merge_state(&mut self, ctx: &Context, bytes: &[u8]) -> bool;
fn as_any(&self) -> &dyn Any;
fn as_any_mut(&mut self) -> &mut dyn Any;
}
impl<C> PlaneCell for ReplicatedCell<C>
where
C: CellCrdt + Serialize + DeserializeOwned + 'static,
C::Value: PartialEq + Clone + 'static,
{
fn merge_state(&mut self, ctx: &Context, bytes: &[u8]) -> bool {
match serde_json::from_slice::<C>(bytes) {
Ok(remote) => self.merge_remote(ctx, &remote),
Err(_) => false,
}
}
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
pub struct CrdtPlaneRuntime {
plane: CrdtPlane,
log: OpLog<CrdtOp>,
cells: BTreeMap<NodeId, Box<dyn PlaneCell>>,
keys: KeyIndex,
families: Vec<Box<dyn FamilyFactory>>,
family_members: BTreeMap<String, Vec<NodeKey>>,
membership_epoch: Option<Source<u64>>,
next_family_node: u64,
}
impl CrdtPlaneRuntime {
pub fn new(peer: PeerId) -> Self {
Self {
plane: CrdtPlane::new(peer),
log: OpLog::new(),
cells: BTreeMap::new(),
keys: KeyIndex::new(),
families: Vec::new(),
family_members: BTreeMap::new(),
membership_epoch: None,
next_family_node: FAMILY_NODE_BASE,
}
}
pub fn peer(&self) -> PeerId {
self.plane.peer()
}
pub fn plane(&self) -> &CrdtPlane {
&self.plane
}
pub fn plane_mut(&mut self) -> &mut CrdtPlane {
&mut self.plane
}
pub fn len(&self) -> usize {
self.cells.len()
}
pub fn is_empty(&self) -> bool {
self.cells.is_empty()
}
pub fn register<C>(&mut self, node: NodeId, key: Option<NodeKey>, cell: ReplicatedCell<C>)
where
C: CellCrdt + Serialize + DeserializeOwned + 'static,
C::Value: PartialEq + Clone + 'static,
{
if let Some(key) = key {
self.keys.insert(key, node);
}
self.cells.insert(node, Box::new(cell));
}
pub fn register_family_lww<V>(&mut self, ctx: &Context, namespace: impl Into<String>)
where
V: Clone + PartialEq + Serialize + DeserializeOwned + 'static,
{
let namespace = namespace.into();
if self.membership_epoch.is_none() {
self.membership_epoch = Some(ctx.source(0u64));
}
self.family_members.entry(namespace.clone()).or_default();
self.families.push(Box::new(LwwFamilyFactory::<V> {
namespace,
_marker: PhantomData,
}));
}
pub fn membership_epoch(&self) -> Option<Source<u64>> {
self.membership_epoch
}
fn mint_family_node(&mut self) -> NodeId {
loop {
let candidate = NodeId(self.next_family_node);
self.next_family_node = self.next_family_node.wrapping_add(1);
if !self.cells.contains_key(&candidate) {
return candidate;
}
}
}
fn bump_membership_epoch(&self, ctx: &Context) {
if let Some(epoch) = self.membership_epoch {
let current = ctx.get(&epoch);
ctx.set(&epoch, current.wrapping_add(1));
}
}
pub fn family_set_lww<V>(
&mut self,
ctx: &Context,
namespace: &str,
key_suffix: &str,
value: V,
now_micros: u64,
) -> Option<CrdtOp>
where
V: Clone + PartialEq + Serialize + DeserializeOwned + 'static,
{
let key = NodeKey::from_segments([namespace, key_suffix]).ok()?;
let stamp = self.plane.tick(now_micros);
let node = match self.keys.node_for_key(&key) {
Some(node) => node,
None => {
let node = self.mint_family_node();
let cell = ReplicatedCell::lww(ctx, value.clone(), stamp);
self.cells.insert(node, Box::new(cell));
self.keys.insert(key.clone(), node);
self.record_family_member(namespace, &key);
self.bump_membership_epoch(ctx);
let state = {
let cell = self
.cells
.get(&node)?
.as_any()
.downcast_ref::<ReplicatedCell<LwwRegister<V>>>()?;
serde_json::to_vec(cell.crdt()).ok()?
};
let op = CrdtOp::keyed(node, key, WireStamp::from(stamp), IpcValue::Inline(state));
self.log.record(stamp, op.clone());
return Some(op);
}
};
let state = {
let cell = self
.cells
.get_mut(&node)?
.as_any_mut()
.downcast_mut::<ReplicatedCell<LwwRegister<V>>>()?;
if !cell.update(ctx, |register| {
register.set(value, stamp);
}) {
return None;
}
serde_json::to_vec(cell.crdt()).ok()?
};
let op = CrdtOp::keyed(node, key, WireStamp::from(stamp), IpcValue::Inline(state));
self.log.record(stamp, op.clone());
Some(op)
}
pub fn family_keys(&self, namespace: &str) -> Vec<NodeKey> {
self.family_members
.get(namespace)
.cloned()
.unwrap_or_default()
}
pub fn family_value_lww<V>(&self, namespace: &str, key_suffix: &str) -> Option<V>
where
V: Clone + PartialEq + 'static,
{
let key = NodeKey::from_segments([namespace, key_suffix]).ok()?;
let node = self.keys.node_for_key(&key)?;
let cell = self
.cells
.get(&node)?
.as_any()
.downcast_ref::<ReplicatedCell<LwwRegister<V>>>()?;
Some(cell.value())
}
fn record_family_member(&mut self, namespace: &str, key: &NodeKey) {
let members = self
.family_members
.entry(namespace.to_string())
.or_default();
if !members.contains(key) {
members.push(key.clone());
}
}
pub fn handle<C>(&self, node: NodeId) -> Option<Source<C::Value>>
where
C: CellCrdt + 'static,
C::Value: PartialEq + Clone + 'static,
{
let cell = self
.cells
.get(&node)?
.as_any()
.downcast_ref::<ReplicatedCell<C>>()?;
Some(cell.handle())
}
pub fn value<C>(&self, node: NodeId) -> Option<C::Value>
where
C: CellCrdt + 'static,
C::Value: PartialEq + Clone + 'static,
{
let cell = self
.cells
.get(&node)?
.as_any()
.downcast_ref::<ReplicatedCell<C>>()?;
Some(cell.value())
}
pub fn local_update<C, F>(
&mut self,
ctx: &Context,
node: NodeId,
now_micros: u64,
mutate: F,
) -> Option<CrdtOp>
where
C: CellCrdt + Serialize + DeserializeOwned + 'static,
C::Value: PartialEq + Clone + 'static,
F: FnOnce(&mut C, HlcStamp),
{
let stamp = self.plane.tick(now_micros);
let state = {
let cell = self
.cells
.get_mut(&node)?
.as_any_mut()
.downcast_mut::<ReplicatedCell<C>>()?;
if !cell.update(ctx, |c| mutate(c, stamp)) {
return None;
}
serde_json::to_vec(cell.crdt()).ok()?
};
let wire = WireStamp::from(stamp);
let op = match self.keys.key_for_node(node).cloned() {
Some(key) => CrdtOp::keyed(node, key, wire, IpcValue::Inline(state)),
None => CrdtOp::new(node, wire, IpcValue::Inline(state)),
};
self.log.record(stamp, op.clone());
Some(op)
}
pub fn ingest(&mut self, ctx: &Context, sync: &CrdtSync, now_micros: u64) -> usize {
for (_, wire) in &sync.frontier {
let stamp = HlcStamp::from(*wire);
if stamp.peer != self.plane.peer() {
self.plane.observe_remote(stamp, now_micros);
}
}
let incoming = sync
.ops
.iter()
.map(|op| (HlcStamp::from(op.stamp), op.clone()));
let Self {
plane,
log,
cells,
keys,
families,
family_members,
membership_epoch,
next_family_node,
} = self;
log.apply_remote(incoming, |stamp, op| {
plane.observe_remote(*stamp, now_micros);
let node = op
.key
.as_ref()
.and_then(|key| keys.node_for_key(key))
.unwrap_or(op.node);
let IpcValue::Inline(bytes) = &op.state else {
return;
};
if let Some(cell) = cells.get_mut(&node) {
cell.merge_state(ctx, bytes);
return;
}
let Some(key) = op.key.as_ref() else { return };
let Some(namespace) = key.segments().next() else {
return;
};
let Some(family) = families.iter().find(|f| f.namespace() == namespace) else {
return;
};
let Some(cell) = family.materialize(ctx, bytes) else {
return;
};
let local_node = loop {
let candidate = NodeId(*next_family_node);
*next_family_node = (*next_family_node).wrapping_add(1);
if !cells.contains_key(&candidate) {
break candidate;
}
};
cells.insert(local_node, cell);
keys.insert(key.clone(), local_node);
let members = family_members.entry(namespace.to_string()).or_default();
if !members.contains(key) {
members.push(key.clone());
}
if let Some(epoch) = membership_epoch.as_ref() {
let current = ctx.get(epoch);
ctx.set(epoch, current.wrapping_add(1));
}
})
}
pub fn wire_frontier(&self) -> Vec<(u64, WireStamp)> {
self.plane
.frontier()
.iter()
.map(|(peer, stamp)| (peer.0, WireStamp::from(stamp)))
.collect()
}
pub fn sync_frame(&self) -> CrdtSync {
self.sync_frame_since(&StampFrontier::new())
}
pub fn sync_frame_since(&self, since: &StampFrontier) -> CrdtSync {
let ops = self
.log
.missing_since(since)
.into_iter()
.map(|(_, op)| op)
.collect();
CrdtSync::new(self.wire_frontier(), ops)
}
pub fn sync_reply(&self, request: &CrdtSync) -> CrdtSync {
self.sync_frame_since(&wire_frontier_to_stamp(&request.frontier))
}
pub fn delta_reply(&self, request: &DeltaSinceRequest) -> CrdtSync {
self.sync_frame_since(&wire_frontier_to_stamp(&request.their_frontier))
}
}
fn wire_frontier_to_stamp(frontier: &[(u64, WireStamp)]) -> StampFrontier {
let mut stamp_frontier = StampFrontier::new();
for (peer, wire) in frontier {
stamp_frontier.observe(PeerId(*peer), HlcStamp::from(*wire));
}
stamp_frontier
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{LwwRegister, MvRegister, PnCounter};
fn lww_cell(ctx: &Context, value: i64) -> ReplicatedCell<LwwRegister<i64>> {
let seed = HlcStamp::from(WireStamp {
wall_time: 0,
logical: 0,
peer: 0,
});
ReplicatedCell::lww(ctx, value, seed)
}
#[test]
fn local_update_emits_keyed_op_and_records_it() {
let ctx = Context::new();
let mut rt = CrdtPlaneRuntime::new(PeerId(1));
let key = NodeKey::new("counter").unwrap();
rt.register(NodeId(7), Some(key.clone()), lww_cell(&ctx, 0));
let op = rt
.local_update::<LwwRegister<i64>, _>(&ctx, NodeId(7), 100, |r, s| {
r.set(42, s);
})
.expect("changed write yields an op");
assert_eq!(op.node, NodeId(7));
assert_eq!(
op.key.as_ref(),
Some(&key),
"producer key projected onto op"
);
assert_eq!(rt.value::<LwwRegister<i64>>(NodeId(7)), Some(42));
assert_eq!(rt.sync_frame().ops.len(), 1);
}
#[test]
fn unchanged_local_write_emits_nothing() {
let ctx = Context::new();
let mut rt = CrdtPlaneRuntime::new(PeerId(1));
rt.register(NodeId(1), None, lww_cell(&ctx, 5));
let op = rt.local_update::<LwwRegister<i64>, _>(&ctx, NodeId(1), 200, |r, s| {
r.set(5, s);
});
assert!(op.is_none(), "a value-preserving write emits no op");
assert_eq!(rt.value::<LwwRegister<i64>>(NodeId(1)), Some(5));
}
#[test]
fn ingest_is_idempotent() {
let ctx_a = Context::new();
let mut a = CrdtPlaneRuntime::new(PeerId(1));
a.register(NodeId(1), None, lww_cell(&ctx_a, 0));
let op = a
.local_update::<LwwRegister<i64>, _>(&ctx_a, NodeId(1), 100, |r, s| {
r.set(11, s);
})
.unwrap();
let ctx_b = Context::new();
let mut b = CrdtPlaneRuntime::new(PeerId(2));
b.register(NodeId(1), None, lww_cell(&ctx_b, 0));
let frame = CrdtSync::new(a.wire_frontier(), vec![op]);
assert_eq!(b.ingest(&ctx_b, &frame, 100), 1, "first apply lands");
assert_eq!(b.ingest(&ctx_b, &frame, 101), 0, "re-apply is a no-op");
assert_eq!(b.value::<LwwRegister<i64>>(NodeId(1)), Some(11));
}
#[test]
fn family_sync_materializes_remote_keys_and_converges() {
let ctx_a = Context::new();
let mut a = CrdtPlaneRuntime::new(PeerId(1));
a.register_family_lww::<bool>(&ctx_a, "live");
let ctx_b = Context::new();
let mut b = CrdtPlaneRuntime::new(PeerId(2));
b.register_family_lww::<bool>(&ctx_b, "live");
let op2 = a
.family_set_lww::<bool>(&ctx_a, "live", "2", true, 100)
.unwrap();
let op3 = a
.family_set_lww::<bool>(&ctx_a, "live", "3", true, 101)
.unwrap();
assert!(b.family_keys("live").is_empty(), "B starts with no entries");
let frame = CrdtSync::new(a.wire_frontier(), vec![op2, op3]);
assert_eq!(
b.ingest(&ctx_b, &frame, 200),
2,
"both remote keys materialize on B"
);
assert_eq!(b.family_keys("live").len(), 2);
assert_eq!(b.family_value_lww::<bool>("live", "2"), Some(true));
assert_eq!(b.family_value_lww::<bool>("live", "3"), Some(true));
assert_eq!(b.ingest(&ctx_b, &frame, 201), 0, "re-apply changes nothing");
let op2b = a
.family_set_lww::<bool>(&ctx_a, "live", "2", false, 300)
.unwrap();
let frame2 = CrdtSync::new(a.wire_frontier(), vec![op2b]);
assert_eq!(b.ingest(&ctx_b, &frame2, 400), 1);
assert_eq!(b.family_value_lww::<bool>("live", "2"), Some(false));
assert_eq!(
b.family_keys("live").len(),
2,
"membership only grows (deferral-not-dealloc)"
);
}
#[test]
fn family_membership_epoch_bumps_on_remote_materialize() {
let ctx_a = Context::new();
let mut a = CrdtPlaneRuntime::new(PeerId(1));
a.register_family_lww::<bool>(&ctx_a, "live");
let op = a
.family_set_lww::<bool>(&ctx_a, "live", "7", true, 100)
.unwrap();
let ctx_b = Context::new();
let mut b = CrdtPlaneRuntime::new(PeerId(2));
b.register_family_lww::<bool>(&ctx_b, "live");
let epoch = b
.membership_epoch()
.expect("registering a family creates the membership epoch");
let before = ctx_b.get(&epoch);
let frame = CrdtSync::new(a.wire_frontier(), vec![op]);
b.ingest(&ctx_b, &frame, 200);
assert_ne!(
before,
ctx_b.get(&epoch),
"materializing a remote key bumps the membership epoch"
);
}
#[test]
fn pn_counter_converges_under_concurrent_increments() {
let ctx_a = Context::new();
let mut a = CrdtPlaneRuntime::new(PeerId(1));
a.register(
NodeId(3),
None,
ReplicatedCell::<PnCounter>::counter(&ctx_a),
);
let ctx_b = Context::new();
let mut b = CrdtPlaneRuntime::new(PeerId(2));
b.register(
NodeId(3),
None,
ReplicatedCell::<PnCounter>::counter(&ctx_b),
);
let op_a = a
.local_update::<PnCounter, _>(&ctx_a, NodeId(3), 100, |c, _| c.increment(PeerId(1), 3))
.unwrap();
let op_b = b
.local_update::<PnCounter, _>(&ctx_b, NodeId(3), 100, |c, _| c.increment(PeerId(2), 5))
.unwrap();
b.ingest(&ctx_b, &CrdtSync::new(a.wire_frontier(), vec![op_a]), 101);
a.ingest(&ctx_a, &CrdtSync::new(b.wire_frontier(), vec![op_b]), 101);
assert_eq!(a.value::<PnCounter>(NodeId(3)), Some(8));
assert_eq!(b.value::<PnCounter>(NodeId(3)), Some(8));
}
#[test]
fn mutual_exchange_expands_membership_and_arms_the_watermark() {
let ctx_a = Context::new();
let mut a = CrdtPlaneRuntime::new(PeerId(1));
a.register(
NodeId(1),
None,
ReplicatedCell::<MvRegister<i64>>::multi_value(&ctx_a),
);
let ctx_b = Context::new();
let mut b = CrdtPlaneRuntime::new(PeerId(2));
b.register(
NodeId(1),
None,
ReplicatedCell::<MvRegister<i64>>::multi_value(&ctx_b),
);
let op_a = a
.local_update::<MvRegister<i64>, _>(&ctx_a, NodeId(1), 100, |r, s| {
r.set(1, s.peer);
})
.unwrap();
let op_b = b
.local_update::<MvRegister<i64>, _>(&ctx_b, NodeId(1), 100, |r, s| {
r.set(2, s.peer);
})
.unwrap();
assert_eq!(a.plane().membership().count(), 1, "B not seen yet");
b.ingest(&ctx_b, &CrdtSync::new(a.wire_frontier(), vec![op_a]), 101);
a.ingest(&ctx_a, &CrdtSync::new(b.wire_frontier(), vec![op_b]), 101);
assert_eq!(
a.plane().membership().count(),
2,
"B folded into membership"
);
assert_eq!(b.plane().membership().count(), 2);
assert!(
a.plane().stability_frontier().is_some(),
"both members observed -> watermark active, GC may run"
);
assert!(b.plane().stability_frontier().is_some());
let mut a_vals = a.value::<MvRegister<i64>>(NodeId(1)).unwrap();
a_vals.sort_unstable();
assert_eq!(a_vals, vec![1, 2]);
}
}