#![deny(missing_docs)]
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::str::FromStr;
use serde::Deserialize;
use serde::Serialize;
use crate::algebra::JoinSemilattice;
use crate::consts::ENTRY_DATA_MAX_LEN;
use crate::dht::Did;
use crate::ecc::HashStr;
use crate::error::Error;
use crate::error::Result;
use crate::message::Encoded;
use crate::message::Encoder;
use crate::message::MessagePayload;
use crate::message::MessageVerificationExt;
mod crdt;
pub use crdt::DataTopicBuffer;
pub use crdt::EntryCrdt;
pub use crdt::EntryDot;
pub use crdt::EntryVersion;
pub use crdt::RelayMessageSet;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum EntryKind {
Data,
RelayMessage,
}
#[derive(Clone, Debug, PartialEq, Eq)]
enum EntryStampKind {
Overwrite,
Delta,
}
#[derive(Serialize)]
struct OperationDigest<'a> {
kind: EntryKind,
did: Did,
data: &'a [Encoded],
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum EntryOperation {
Overwrite(Entry),
Extend(Entry),
Touch(Entry),
Tombstone(Entry),
CompactData(Entry),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PlacedEntryOperation {
pub placement: Did,
pub op: EntryOperation,
}
impl PlacedEntryOperation {
pub fn entry_key(&self) -> Result<Did> {
self.op.did()
}
pub fn placement_belongs_to_entry(&self, redundancy: u16) -> Result<bool> {
let entry_key = self.entry_key()?;
placement_belongs_to_entry_key(entry_key, self.placement, redundancy)
}
pub fn validate_placement(&self, redundancy: u16) -> Result<()> {
if self.placement_belongs_to_entry(redundancy)? {
return Ok(());
}
Err(Error::InvalidMessage(
"placed entry operation targets a placement outside the entry's affine replica set"
.to_string(),
))
}
}
fn placement_belongs_to_entry_key(entry_key: Did, placement: Did, redundancy: u16) -> Result<bool> {
Ok(entry_key.rotate_affine(redundancy)?.contains(&placement))
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Entry {
pub did: Did,
pub data: Vec<Encoded>,
pub kind: EntryKind,
#[serde(default)]
pub crdt: EntryCrdt,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PlacedEntry {
pub key: Did,
pub entry: Entry,
}
impl PlacedEntry {
pub fn new(key: Did, entry: Entry) -> Self {
Self { key, entry }
}
pub fn placement_belongs_to_entry(&self, redundancy: u16) -> Result<bool> {
placement_belongs_to_entry_key(self.entry.did, self.key, redundancy)
}
pub fn validate_placement(&self, redundancy: u16) -> Result<()> {
if self.placement_belongs_to_entry(redundancy)? {
return Ok(());
}
Err(Error::InvalidMessage(
"synced placed entry targets a placement outside the entry's affine replica set"
.to_string(),
))
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncedEntryAck {
pub key: Did,
pub entry: Entry,
}
impl SyncedEntryAck {
pub fn new(key: Did, entry: Entry) -> Self {
Self { key, entry }
}
pub fn confirms_local_value(&self, local: &Entry) -> Result<bool> {
Ok(self.entry.clone().try_into_storage_entry()?
== local.clone().try_into_storage_entry()?)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EntryLookupKey {
pub resource: Did,
pub placement: Did,
}
impl EntryLookupKey {
pub fn new(resource: Did, placement: Did) -> Self {
Self {
resource,
placement,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct PlacementMiss {
pub key: Did,
pub owner: Did,
}
impl PlacementMiss {
pub fn new(key: Did, owner: Did) -> Self {
Self { key, owner }
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EntryLookupEvidence {
pub entry: Entry,
pub misses: Vec<PlacementMiss>,
}
impl EntryLookupEvidence {
pub fn new(entry: Entry, misses: Vec<PlacementMiss>) -> Self {
Self { entry, misses }
}
}
impl Entry {
pub fn new(did: Did, data: Vec<Encoded>, kind: EntryKind) -> Self {
Self {
did,
data,
kind,
crdt: EntryCrdt::default(),
}
}
pub fn gen_did(topic: &str) -> Result<Did> {
let hash: HashStr = topic.into();
let did = Did::from_str(&hash.inner());
tracing::debug!("gen_did: topic: {}, did: {:?}", topic, did);
did
}
}
impl EntryOperation {
pub fn stamped(self, actor: Did) -> Result<Self> {
Ok(match self {
EntryOperation::Overwrite(entry) => EntryOperation::Overwrite(
entry.ensure_stamp_after(actor, None, EntryStampKind::Overwrite)?,
),
EntryOperation::Extend(entry) => EntryOperation::Extend(entry.ensure_stamp_after(
actor,
None,
EntryStampKind::Delta,
)?),
EntryOperation::Touch(entry) => EntryOperation::Touch(entry.ensure_stamp_after(
actor,
None,
EntryStampKind::Delta,
)?),
EntryOperation::Tombstone(entry) => EntryOperation::Tombstone(entry),
EntryOperation::CompactData(entry) => {
EntryOperation::CompactData(entry.ensure_overwrite_stamp_after(actor, None)?)
}
})
}
pub fn did(&self) -> Result<Did> {
Ok(match self {
EntryOperation::Overwrite(entry) => entry.did,
EntryOperation::Extend(entry) => entry.did,
EntryOperation::Touch(entry) => entry.did,
EntryOperation::Tombstone(entry) => entry.did,
EntryOperation::CompactData(entry) => entry.did,
})
}
pub fn kind(&self) -> EntryKind {
match self {
EntryOperation::Overwrite(entry) => entry.kind,
EntryOperation::Extend(entry) => entry.kind,
EntryOperation::Touch(entry) => entry.kind,
EntryOperation::Tombstone(entry) => entry.kind,
EntryOperation::CompactData(entry) => entry.kind,
}
}
pub fn gen_default_entry(self) -> Result<Entry> {
Ok(Entry::new(self.did()?, vec![], self.kind()))
}
}
impl TryFrom<MessagePayload> for Entry {
type Error = Error;
fn try_from(msg: MessagePayload) -> Result<Self> {
let did = msg.signer() + Did::from(1u32);
let data = msg.encode()?;
Ok(Self {
did,
data: vec![data],
kind: EntryKind::RelayMessage,
crdt: EntryCrdt::default(),
})
}
}
impl TryFrom<(String, Encoded)> for Entry {
type Error = Error;
fn try_from((topic, e): (String, Encoded)) -> Result<Self> {
Ok(Self {
did: Self::gen_did(&topic)?,
data: vec![e],
kind: EntryKind::Data,
crdt: EntryCrdt::default(),
})
}
}
impl TryFrom<(String, String)> for Entry {
type Error = Error;
fn try_from((topic, s): (String, String)) -> Result<Self> {
let encoded_message = s.encode()?;
(topic, encoded_message).try_into()
}
}
impl TryFrom<String> for Entry {
type Error = Error;
fn try_from(topic: String) -> Result<Self> {
(topic.clone(), topic).try_into()
}
}
impl Entry {
fn with_element_dots(mut self, version: EntryVersion) -> Result<Self> {
self.crdt.dots = self
.data
.iter()
.enumerate()
.map(|(index, _)| EntryDot::for_index(version, index))
.collect::<Result<Vec<_>>>()?;
Ok(self)
}
fn stamp_overwrite(mut self, version: EntryVersion) -> Result<Self> {
self.crdt.register = Some(version);
self.with_element_dots(version)
}
fn stamp_delta(self, version: EntryVersion) -> Result<Self> {
self.with_element_dots(version)
}
fn stamp(self, version: EntryVersion, kind: EntryStampKind) -> Result<Self> {
match kind {
EntryStampKind::Overwrite => self.stamp_overwrite(version),
EntryStampKind::Delta => self.stamp_delta(version),
}
}
fn operation_digest(&self) -> Result<Did> {
let digest = OperationDigest {
kind: self.kind,
did: self.did,
data: &self.data,
};
let bytes = rings_codec::serialize(&digest).map_err(Error::CodecSerialize)?;
Did::try_from(HashStr::from_bytes(&bytes))
}
fn issue_version_after(&self, actor: Did, floor: Option<EntryVersion>) -> Result<EntryVersion> {
Ok(EntryVersion::issued_by(actor, self.operation_digest()?).after(floor))
}
fn ensure_stamp_after(
self,
actor: Did,
floor: Option<EntryVersion>,
kind: EntryStampKind,
) -> Result<Self> {
match self.crdt.has_write_witness() {
true => Ok(self),
false => {
let version = self.issue_version_after(actor, floor)?;
self.stamp(version, kind)
}
}
}
fn ensure_overwrite_stamp_after(self, actor: Did, floor: Option<EntryVersion>) -> Result<Self> {
match self.crdt.register.is_some() {
true => Ok(self),
false => {
let version = self.issue_version_after(actor, floor)?;
self.stamp_overwrite(version)
}
}
}
fn max_observed_version(&self) -> Option<EntryVersion> {
self.crdt
.dots
.iter()
.map(|dot| dot.version)
.chain(self.crdt.tombstones.iter().map(|dot| dot.version))
.chain(self.crdt.register)
.max()
}
fn validate_same_carrier(&self, other: &Self) -> Result<()> {
if !self.same_kind_as(other) {
return Err(Error::EntryKindNotEqual);
}
if !self.same_key_as(other) {
return Err(Error::EntryDidNotEqual);
}
Ok(())
}
fn dot_for_element(&self, index: usize) -> Result<EntryDot> {
if let Some(dot) = self.crdt.dots.get(index).copied() {
return Ok(dot);
}
EntryDot::for_index(self.crdt.legacy_floor(), index)
}
fn topic_buffer(&self) -> Result<DataTopicBuffer> {
let mut values = BTreeMap::new();
for (index, value) in self.data.iter().cloned().enumerate() {
let dot = self.dot_for_element(index)?;
values
.entry(value)
.and_modify(|current: &mut EntryDot| {
*current = (*current).max(dot);
})
.or_insert(dot);
}
Ok(DataTopicBuffer::new(
self.crdt.register,
values,
self.crdt.tombstones.iter().copied().collect(),
))
}
fn relay_set(&self) -> Result<RelayMessageSet> {
Ok(RelayMessageSet::new(
self.topic_buffer()?,
self.crdt.tombstones.iter().copied().collect(),
))
}
fn materialize_elements(
did: Did,
kind: EntryKind,
register: Option<EntryVersion>,
elements: impl IntoIterator<Item = (Encoded, EntryDot)>,
tombstones: BTreeSet<EntryDot>,
) -> Self {
let mut visible = elements
.into_iter()
.filter(|(_, dot)| {
let visible_after_reset = register.is_none_or(|floor| dot.version >= floor);
visible_after_reset && !tombstones.contains(dot)
})
.collect::<Vec<_>>();
visible.sort_by(|(left_value, left_dot), (right_value, right_dot)| {
left_dot
.cmp(right_dot)
.then_with(|| left_value.cmp(right_value))
});
let skip_count = visible.len().saturating_sub(ENTRY_DATA_MAX_LEN);
let visible = visible.into_iter().skip(skip_count).collect::<Vec<_>>();
let (data, dots): (Vec<_>, Vec<_>) = visible.into_iter().unzip();
Self {
did,
data,
kind,
crdt: EntryCrdt {
register,
dots,
tombstones: tombstones.into_iter().collect(),
},
}
}
fn materialize_topic_buffer(&self, buffer: DataTopicBuffer) -> Self {
Self::materialize_elements(
self.did,
self.kind,
buffer.register,
buffer.values,
buffer.removes,
)
}
fn materialize_relay_set(&self, set: RelayMessageSet) -> Self {
Self::materialize_elements(
self.did,
self.kind,
set.adds.register,
set.adds.values,
set.removes,
)
}
fn compacted_data_dot(floor: EntryVersion, value: &Encoded) -> Result<EntryDot> {
let operation = Did::try_from(HashStr::from_bytes(value.value().as_bytes()))?;
let version =
EntryVersion::new(floor.logical_time_ms, floor.actor, operation).after(Some(floor));
EntryDot::for_index(version, 0)
}
fn compact_data_element(
floor: EntryVersion,
removal_values: &BTreeSet<Encoded>,
value: Encoded,
dot: EntryDot,
) -> Result<Option<(Encoded, EntryDot)>> {
match dot.version < floor {
true if removal_values.contains(&value) => Ok(None),
true => Self::compacted_data_dot(floor, &value).map(|dot| Some((value, dot))),
false => Ok(Some((value, dot))),
}
}
fn data_compaction_candidates(
payload_order: &[Encoded],
live_values: BTreeMap<Encoded, EntryDot>,
) -> Vec<(Encoded, EntryDot)> {
let (ordered_values, remaining_values) = payload_order.iter().fold(
(Vec::new(), live_values),
|(mut ordered, mut remaining), value| {
if let Some(dot) = remaining.remove(value) {
ordered.push((value.clone(), dot));
}
(ordered, remaining)
},
);
ordered_values.into_iter().chain(remaining_values).collect()
}
fn compact_data_elements(
floor: EntryVersion,
removal_values: &BTreeSet<Encoded>,
values: impl IntoIterator<Item = (Encoded, EntryDot)>,
) -> Result<Vec<(Encoded, EntryDot)>> {
values.into_iter().try_fold(
Vec::new(),
|mut elements, (value, dot)| -> Result<Vec<(Encoded, EntryDot)>> {
match Self::compact_data_element(floor, removal_values, value, dot)? {
Some(element) => {
elements.push(element);
Ok(elements)
}
None => Ok(elements),
}
},
)
}
fn compact_data_output_floor(
current_floor: Option<EntryVersion>,
operation_floor: EntryVersion,
) -> EntryVersion {
current_floor.map_or(operation_floor, |current| current.max(operation_floor))
}
fn compact_data_tombstones(
floor: EntryVersion,
tombstones: BTreeSet<EntryDot>,
) -> BTreeSet<EntryDot> {
tombstones
.into_iter()
.filter(|dot| dot.version >= floor)
.collect()
}
pub fn join(&self, other: Self) -> Result<Self> {
self.validate_same_carrier(&other)?;
match self.kind {
EntryKind::Data => {
Ok(self.materialize_topic_buffer(self.topic_buffer()?.join(other.topic_buffer()?)))
}
EntryKind::RelayMessage => {
Ok(self.materialize_relay_set(self.relay_set()?.join(other.relay_set()?)))
}
}
}
pub fn affine(&self, scalar: u16) -> Result<Vec<Entry>> {
Ok(self
.did
.rotate_affine(scalar)?
.into_iter()
.map(|did| self.clone_with_did(did))
.collect())
}
pub fn clone_with_did(&self, did: Did) -> Self {
let mut entry = self.clone();
entry.did = did;
entry
}
fn is_data_entry(&self) -> bool {
self.kind == EntryKind::Data
}
fn same_kind_as(&self, other: &Self) -> bool {
self.kind == other.kind
}
fn same_key_as(&self, other: &Self) -> bool {
self.did == other.did
}
pub fn try_into_storage_entry(self) -> Result<Self> {
match self.kind {
EntryKind::Data => {
let buffer = self.topic_buffer()?;
Ok(self.materialize_topic_buffer(buffer))
}
EntryKind::RelayMessage => {
let set = self.relay_set()?;
Ok(self.materialize_relay_set(set))
}
}
}
pub fn operate(&self, op: EntryOperation, actor: Did) -> Result<Self> {
match op {
EntryOperation::Overwrite(entry) => self.overwrite(entry, actor),
EntryOperation::Extend(entry) => self.extend(entry, actor),
EntryOperation::Touch(entry) => self.touch(entry, actor),
EntryOperation::Tombstone(entry) => self.tombstone(entry),
EntryOperation::CompactData(entry) => self.compact_data(entry, actor),
}
}
pub fn overwrite(&self, other: Self, actor: Did) -> Result<Self> {
if !self.is_data_entry() {
return Err(Error::EntryNotOverwritable);
}
self.join(other.ensure_stamp_after(
actor,
self.max_observed_version(),
EntryStampKind::Overwrite,
)?)
}
pub fn extend(&self, other: Self, actor: Did) -> Result<Self> {
if !self.is_data_entry() {
return Err(Error::EntryNotAppendable);
}
self.join(other.ensure_stamp_after(
actor,
self.max_observed_version(),
EntryStampKind::Delta,
)?)
}
pub fn touch(&self, other: Self, actor: Did) -> Result<Self> {
if !self.is_data_entry() {
return Err(Error::EntryNotAppendable);
}
self.join(other.ensure_stamp_after(
actor,
self.max_observed_version(),
EntryStampKind::Delta,
)?)
}
pub fn tombstone(&self, other: Self) -> Result<Self> {
self.validate_same_carrier(&other)?;
let target_values = other.data.into_iter().collect::<BTreeSet<_>>();
let target_dots = other.crdt.dots.into_iter().collect::<BTreeSet<_>>();
let has_dot_witness = !target_dots.is_empty();
match self.kind {
EntryKind::Data => {
let mut buffer = self.topic_buffer()?;
for (value, dot) in &buffer.values {
if target_dots.contains(dot)
|| (!has_dot_witness && target_values.contains(value))
{
buffer.removes.insert(*dot);
}
}
Ok(self.materialize_topic_buffer(buffer))
}
EntryKind::RelayMessage => {
let mut set = self.relay_set()?;
for (value, dot) in &set.adds.values {
if target_dots.contains(dot)
|| (!has_dot_witness && target_values.contains(value))
{
set.removes.insert(*dot);
}
}
Ok(self.materialize_relay_set(set))
}
}
}
pub fn compact_data(&self, removals: Self, actor: Did) -> Result<Self> {
match self.is_data_entry() {
true => self.compact_data_entry(removals, actor),
false => Err(Error::EntryNotOverwritable),
}
}
fn compact_data_entry(&self, removals: Self, actor: Did) -> Result<Self> {
let removals = removals.ensure_overwrite_stamp_after(actor, self.max_observed_version())?;
self.validate_same_carrier(&removals)?;
let floor = removals.crdt.register.ok_or_else(|| {
Error::InvalidMessage("compact data operation has no register floor".to_string())
})?;
let removal_values = removals.data.into_iter().collect::<BTreeSet<_>>();
let buffer = self.topic_buffer()?;
let output_floor = Self::compact_data_output_floor(self.crdt.register, floor);
let elements = Self::compact_data_elements(
floor,
&removal_values,
Self::data_compaction_candidates(&self.data, buffer.values),
)?;
let tombstones = Self::compact_data_tombstones(output_floor, buffer.removes);
Ok(Self::materialize_elements(
self.did,
EntryKind::Data,
Some(output_floor),
elements,
tombstones,
))
}
}
#[cfg(test)]
mod test_entry;