use std::sync::Arc;
use dynomite::cluster::ReplicaTarget;
use dynomite::embed::hooks::BoxFuture;
use dynomite::hashkit::{hash64, HashType};
use dynomite::msg::ConsistencyLevel;
use crate::bucket_props::{BucketProps, BucketPropsRegistry};
use crate::datatypes::keyfun::KeyFun;
use crate::replication::{plan_replicas, ReplicationPlan, ReplicationStrategy, RingView};
#[derive(Clone, Debug)]
pub struct RouteDecision {
pub bucket_type: Vec<u8>,
pub props: BucketProps,
pub route_bytes: Vec<u8>,
pub key_hash: u64,
pub plan: ReplicationPlan,
}
impl RouteDecision {
#[must_use]
pub fn keyfun(&self) -> KeyFun {
self.props.effective_keyfun()
}
#[must_use]
pub fn strategy(&self) -> ReplicationStrategy {
self.props.effective_strategy()
}
#[must_use]
pub fn replica_list(&self) -> Vec<ReplicaTarget> {
self.plan.clone().into_replica_list()
}
#[must_use]
pub fn primary_peer_idx(&self) -> Option<u32> {
match &self.plan {
ReplicationPlan::Successors { primary, .. } => Some(primary.peer_idx),
ReplicationPlan::Topology(targets) => targets.first().map(|t| t.peer_idx),
}
}
#[must_use]
pub fn primary_replica_count(&self) -> usize {
self.replica_list()
.iter()
.filter(|t| !t.is_fallback)
.count()
}
}
#[derive(Clone, Debug)]
pub struct BucketRouter {
registry: Arc<BucketPropsRegistry>,
ring: Arc<RingView>,
hash: HashType,
liveness: Option<Arc<dyn crate::replication::ReplicaLiveness>>,
#[cfg(feature = "wasm")]
keyfun_store: Option<crate::datatypes::keyfun_wasm::WasmKeyfunStore>,
}
impl BucketRouter {
#[must_use]
pub fn new(registry: Arc<BucketPropsRegistry>, ring: Arc<RingView>, hash: HashType) -> Self {
Self {
registry,
ring,
hash,
liveness: None,
#[cfg(feature = "wasm")]
keyfun_store: None,
}
}
#[must_use]
pub fn with_liveness(mut self, liveness: Arc<dyn crate::replication::ReplicaLiveness>) -> Self {
self.liveness = Some(liveness);
self
}
#[cfg(feature = "wasm")]
#[must_use]
pub fn with_keyfun_store(
mut self,
store: crate::datatypes::keyfun_wasm::WasmKeyfunStore,
) -> Self {
self.keyfun_store = Some(store);
self
}
#[cfg(feature = "wasm")]
#[must_use]
pub fn keyfun_store(&self) -> Option<&crate::datatypes::keyfun_wasm::WasmKeyfunStore> {
self.keyfun_store.as_ref()
}
#[must_use]
pub fn registry(&self) -> &Arc<BucketPropsRegistry> {
&self.registry
}
#[must_use]
pub fn ring(&self) -> &Arc<RingView> {
&self.ring
}
#[must_use]
pub fn hash_type(&self) -> HashType {
self.hash
}
#[must_use]
pub fn route(&self, bucket_type: &[u8], bucket: &[u8], key: &[u8]) -> RouteDecision {
self.try_route(bucket_type, bucket, key).expect(
"invariant: route called on a Custom keyfun without a keyfun store; use try_route",
)
}
pub fn try_route(
&self,
bucket_type: &[u8],
bucket: &[u8],
key: &[u8],
) -> Result<RouteDecision, crate::datatypes::keyfun::KeyFunError> {
let props = self.registry.resolve(bucket_type, bucket);
let kf = props.effective_keyfun();
let strategy = props.effective_strategy();
let n_val = props.effective_n_val();
let route_bytes = self.resolve_route_bytes(&kf, bucket, key)?;
let key_hash = hash64(self.hash, &route_bytes);
let plan = match (&self.liveness, strategy) {
(Some(liveness), ReplicationStrategy::Successors) => {
crate::replication::plan_replicas_with_liveness(
self.ring.as_ref(),
key_hash,
n_val,
liveness.as_ref(),
)
}
_ => plan_replicas(
self.ring.as_ref(),
key_hash,
n_val,
strategy,
ConsistencyLevel::DcOne,
),
};
Ok(RouteDecision {
bucket_type: if bucket_type.is_empty() {
b"default".to_vec()
} else {
bucket_type.to_vec()
},
props,
route_bytes,
key_hash,
plan,
})
}
fn resolve_route_bytes(
&self,
kf: &KeyFun,
bucket: &[u8],
key: &[u8],
) -> Result<Vec<u8>, crate::datatypes::keyfun::KeyFunError> {
match kf {
KeyFun::Std | KeyFun::BucketOnly => kf.try_route_bytes(bucket, key),
KeyFun::Custom(module_id) => self.resolve_custom_route_bytes(module_id, bucket, key),
}
}
#[cfg(feature = "wasm")]
fn resolve_custom_route_bytes(
&self,
module_id: &str,
bucket: &[u8],
key: &[u8],
) -> Result<Vec<u8>, crate::datatypes::keyfun::KeyFunError> {
match &self.keyfun_store {
Some(store) => store.route_bytes(module_id, bucket, key),
None => Err(crate::datatypes::keyfun::KeyFunError::ModuleNotFound(
module_id.to_string(),
)),
}
}
#[cfg(not(feature = "wasm"))]
fn resolve_custom_route_bytes(
&self,
module_id: &str,
_bucket: &[u8],
_key: &[u8],
) -> Result<Vec<u8>, crate::datatypes::keyfun::KeyFunError> {
let _ = self;
Err(crate::datatypes::keyfun::KeyFunError::ModuleNotFound(
module_id.to_string(),
))
}
}
pub const ACK_STORED: u8 = 1;
pub const ACK_STORED_DURABLE: u8 = 2;
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum PeerOp {
Put {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
value: Vec<u8>,
},
Get {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
},
Del {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
},
DtUpdate {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
op: Vec<u8>,
},
DtFetch {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
tag: u8,
},
RepairPut {
bucket_type: Vec<u8>,
bucket: Vec<u8>,
key: Vec<u8>,
storage: Vec<u8>,
},
}
#[must_use]
pub fn composite_storage_bucket(bucket_type: &[u8], bucket: &[u8]) -> Vec<u8> {
let ty: &[u8] = if bucket_type.is_empty() || bucket_type == b"default" {
b"default"
} else {
bucket_type
};
let mut out = Vec::with_capacity(ty.len() + 1 + bucket.len());
out.extend_from_slice(ty);
out.push(0x1f);
out.extend_from_slice(bucket);
out
}
#[must_use]
pub fn split_composite_storage_bucket(composite: &[u8]) -> (&[u8], &[u8]) {
match composite.iter().position(|&b| b == 0x1f) {
Some(i) => (&composite[..i], &composite[i + 1..]),
None => (b"default", composite),
}
}
pub trait PeerOutbound: Send + Sync + std::fmt::Debug {
fn dispatch(&self, peer_idx: u32, op: PeerOp) -> BoxFuture<'_, ()>;
fn request(&self, peer_idx: u32, op: PeerOp) -> BoxFuture<'_, Option<Vec<u8>>> {
let _ = (peer_idx, op);
Box::pin(async { None })
}
}
pub trait PrecommitRunner: Send + Sync + std::fmt::Debug {
fn run(&self, module_id: &str, value: &[u8]) -> Result<Vec<u8>, PrecommitVeto>;
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum PrecommitVeto {
Rejected(String),
Error(String),
}
pub trait PostcommitRunner: Send + Sync + std::fmt::Debug {
fn run(&self, module_id: &str, value: &[u8]);
}
#[derive(Clone, Debug)]
pub struct RoutingHooks {
pub router: Arc<BucketRouter>,
pub outbound: Arc<dyn PeerOutbound>,
pub local_actor: crate::datatypes::ActorId,
pub local_peer_idx: u32,
pub precommit: Option<Arc<dyn PrecommitRunner>>,
pub postcommit: Option<Arc<dyn PostcommitRunner>>,
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use crate::bucket_props::BucketProps;
fn five_peer_ring() -> Arc<RingView> {
let span = u64::from(u32::MAX);
let pts: Vec<RingPoint> = (0..5u32)
.map(|i| RingPoint::new(u64::from(i) * span / 5, i, "dc1", "r1"))
.collect();
Arc::new(RingView::new(pts))
}
use crate::replication::RingPoint;
fn router_with_bucket(props: BucketProps) -> BucketRouter {
let reg = Arc::new(BucketPropsRegistry::new_riak_defaults());
reg.set(b"default", b"users", props);
BucketRouter::new(reg, five_peer_ring(), HashType::Murmur)
}
#[test]
fn bucketonly_keyfun_collapses_keys_to_one_partition() {
let router = router_with_bucket(BucketProps {
keyfun: Some(KeyFun::BucketOnly),
strategy: Some(ReplicationStrategy::Successors),
n_val: Some(3),
..BucketProps::default()
});
let mut buckets: HashMap<u32, usize> = HashMap::new();
for i in 0..100u32 {
let key = format!("key-{i}");
let d = router.route(b"default", b"users", key.as_bytes());
let primary = d.primary_peer_idx().expect("successors yields primary");
*buckets.entry(primary).or_insert(0) += 1;
}
assert_eq!(
buckets.len(),
1,
"BUCKETONLY routes every key to one peer; saw {buckets:?}"
);
}
#[test]
fn std_keyfun_distributes_within_5_percent_of_uniform() {
let router = router_with_bucket(BucketProps {
keyfun: Some(KeyFun::Std),
strategy: Some(ReplicationStrategy::Successors),
n_val: Some(1),
..BucketProps::default()
});
let mut buckets: HashMap<u32, usize> = HashMap::new();
let total: u32 = 10_000;
for i in 0..total {
let key = format!("key-{i}");
let d = router.route(b"default", b"users", key.as_bytes());
let primary = d.primary_peer_idx().expect("successors yields primary");
*buckets.entry(primary).or_insert(0) += 1;
}
let expected = f64::from(total) / 5.0;
let tolerance = expected * 0.05;
for peer in 0..5u32 {
let observed = f64::from(u32::try_from(*buckets.get(&peer).unwrap_or(&0)).unwrap());
let delta = (observed - expected).abs();
assert!(
delta < tolerance,
"peer {peer}: observed {observed}, expected {expected:.0}, delta {delta:.1} >= tol {tolerance:.1}"
);
}
}
#[test]
fn route_bytes_match_keyfun_shape() {
let router = router_with_bucket(BucketProps {
keyfun: Some(KeyFun::BucketOnly),
..BucketProps::default()
});
let d = router.route(b"default", b"users", b"alice");
assert_eq!(d.route_bytes, b"users");
let router = router_with_bucket(BucketProps {
keyfun: Some(KeyFun::Std),
..BucketProps::default()
});
let d = router.route(b"default", b"users", b"alice");
assert_eq!(d.route_bytes, b"users/alice");
}
#[test]
fn topology_strategy_yields_empty_replica_list() {
let router = router_with_bucket(BucketProps {
strategy: Some(ReplicationStrategy::Topology),
..BucketProps::default()
});
let d = router.route(b"default", b"users", b"alice");
assert!(d.replica_list().is_empty());
assert!(matches!(d.plan, ReplicationPlan::Topology(_)));
}
#[test]
fn empty_bucket_type_normalises_to_default() {
let router = router_with_bucket(BucketProps {
keyfun: Some(KeyFun::BucketOnly),
..BucketProps::default()
});
let d = router.route(b"", b"users", b"alice");
assert_eq!(d.bucket_type, b"default");
assert_eq!(d.keyfun(), KeyFun::BucketOnly);
}
}