use std::{
collections::{HashMap, HashSet, VecDeque},
sync::{Arc, RwLock},
};
use tracing::instrument;
use crate::primitives::{AccountId, AccountSetId, JournalId};
use super::{
error::AccountSetError,
graph_validation::{
has_duplicate_account_membership_paths, validate_set_memberships, AccountMembership,
SetMembership,
},
repo::{AccountSetRepo, DirectMembershipProbe, SetGraphNode},
};
const TIMER_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
const COLD_EPOCH: i64 = -1;
#[derive(Debug, Clone, Copy)]
struct SetMeta {
journal_id: JournalId,
eventually_consistent: bool,
}
#[derive(Debug)]
struct GraphSnapshot {
epoch: i64,
parents: HashMap<AccountSetId, Vec<AccountSetId>>,
children: HashMap<AccountSetId, Vec<AccountSetId>>,
meta: HashMap<AccountSetId, SetMeta>,
}
impl GraphSnapshot {
fn cold() -> Self {
Self {
epoch: COLD_EPOCH,
parents: HashMap::new(),
children: HashMap::new(),
meta: HashMap::new(),
}
}
fn edges_connected_to(&self, members: &[SetMembership]) -> Vec<SetMembership> {
let mut pending: Vec<_> = members
.iter()
.flat_map(|edge| [edge.account_set_id, edge.member_account_set_id])
.collect();
let mut visited = HashSet::new();
let mut edges = HashSet::new();
while let Some(account_set_id) = pending.pop() {
if !visited.insert(account_set_id) {
continue;
}
for parent_id in self.parents.get(&account_set_id).into_iter().flatten() {
edges.insert(SetMembership {
account_set_id: *parent_id,
member_account_set_id: account_set_id,
});
pending.push(*parent_id);
}
for member_id in self.children.get(&account_set_id).into_iter().flatten() {
edges.insert(SetMembership {
account_set_id,
member_account_set_id: *member_id,
});
pending.push(*member_id);
}
}
edges.into_iter().collect()
}
}
struct Overlay {
parents: HashMap<AccountSetId, Vec<AccountSetId>>,
meta: HashMap<AccountSetId, SetMeta>,
}
struct IndexedNodes {
parents: HashMap<AccountSetId, Vec<AccountSetId>>,
children: HashMap<AccountSetId, Vec<AccountSetId>>,
meta: HashMap<AccountSetId, SetMeta>,
}
#[derive(Clone)]
pub(super) struct SetGraphCache {
inner: Arc<SetGraphCacheInner>,
}
struct SetGraphCacheInner {
repo: AccountSetRepo,
snapshot: RwLock<Arc<GraphSnapshot>>,
refresh_lock: tokio::sync::Mutex<()>,
}
impl std::fmt::Debug for SetGraphCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SetGraphCache").finish_non_exhaustive()
}
}
impl SetGraphCache {
pub(super) fn new(repo: AccountSetRepo) -> Self {
let inner = Arc::new(SetGraphCacheInner {
repo,
snapshot: RwLock::new(Arc::new(GraphSnapshot::cold())),
refresh_lock: tokio::sync::Mutex::new(()),
});
let weak = Arc::downgrade(&inner);
tokio::spawn(async move {
let mut interval = tokio::time::interval(TIMER_REFRESH_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
loop {
interval.tick().await;
let Some(inner) = weak.upgrade() else { break };
if let Err(error) = Self::refresh(&inner).await {
tracing::warn!(%error, "set_graph_cache timer refresh failed");
}
}
});
Self { inner }
}
#[instrument(
level = "debug",
name = "account_set.resolve_from_probe_in_op",
skip(self, op, probe_seeds, entry_pairs),
fields(accounts = entry_pairs.0.len(), path = tracing::field::Empty),
err(level = "warn")
)]
pub(super) async fn resolve_from_probe_in_op(
&self,
op: &mut impl es_entity::AtomicOperation,
journal_id: JournalId,
probe_epoch: i64,
probe_seeds: &[AccountMembership],
entry_pairs: &(Vec<AccountId>, Vec<&str>),
) -> Result<HashMap<AccountId, Vec<AccountSetId>>, AccountSetError> {
let span = tracing::Span::current();
let probe = DirectMembershipProbe {
epoch: probe_epoch,
seeds: probe_seeds.to_vec(),
};
if probe.seeds.is_empty() {
span.record("path", "no_memberships");
return Ok(HashMap::new());
}
let snapshot = self.load();
if snapshot.epoch != probe.epoch {
span.record(
"path",
if snapshot.epoch == COLD_EPOCH {
"fallback_cold"
} else {
"fallback_epoch"
},
);
self.spawn_refresh();
return self
.inner
.repo
.walk_mappings_and_lock_in_op(&mut *op, journal_id, entry_pairs)
.await;
}
let missing: Vec<AccountSetId> = {
let mut missing: Vec<_> = probe
.seeds
.iter()
.map(|seed| seed.account_set_id)
.filter(|set_id| !snapshot.meta.contains_key(set_id))
.collect();
missing.sort_unstable();
missing.dedup();
missing
};
let overlay = if missing.is_empty() {
None
} else {
let nodes = self
.inner
.repo
.fetch_set_graph_nodes_in_op(op, &missing)
.await?;
let IndexedNodes { parents, meta, .. } = Self::index_nodes(nodes);
Some(Overlay { parents, meta })
};
match Self::expand(
&snapshot,
overlay.as_ref(),
journal_id,
&probe.seeds,
entry_pairs,
) {
Some((mappings, lock_pairs)) => {
span.record(
"path",
if overlay.is_some() {
"supplement"
} else {
"memory"
},
);
self.inner
.repo
.lock_resolved_ancestors_in_op(op, journal_id, &lock_pairs)
.await?;
Ok(mappings)
}
None => {
span.record("path", "fallback_unknown");
self.inner
.repo
.walk_mappings_and_lock_in_op(&mut *op, journal_id, entry_pairs)
.await
}
}
}
#[instrument(
level = "debug",
name = "account_set.assert_no_double_membership",
skip(self, op, members),
fields(pairs = members.len(), path = tracing::field::Empty),
err(level = "warn")
)]
pub(super) async fn assert_no_double_membership_in_op(
&self,
op: &mut impl es_entity::AtomicOperation,
members: &[AccountMembership],
) -> Result<(), AccountSetError> {
let span = tracing::Span::current();
let distinct_account_ids: Vec<AccountId> = {
let mut ids: Vec<AccountId> = members.iter().map(|m| m.account_id).collect();
ids.sort_unstable();
ids.dedup();
ids
};
let probe = self
.inner
.repo
.probe_direct_memberships_in_op(op, &distinct_account_ids)
.await?;
let snapshot = self.load();
if snapshot.epoch != probe.epoch {
span.record(
"path",
if snapshot.epoch == COLD_EPOCH {
"fallback_cold"
} else {
"fallback_epoch"
},
);
self.spawn_refresh();
return self
.inner
.repo
.assert_no_double_membership(op, members)
.await;
}
let missing: Vec<AccountSetId> = {
let mut missing: Vec<_> = members
.iter()
.chain(probe.seeds.iter())
.map(|membership| membership.account_set_id)
.filter(|set_id| !snapshot.meta.contains_key(set_id))
.collect();
missing.sort_unstable();
missing.dedup();
missing
};
let overlay = if missing.is_empty() {
None
} else {
let nodes = self
.inner
.repo
.fetch_set_graph_nodes_in_op(op, &missing)
.await?;
let IndexedNodes { parents, meta, .. } = Self::index_nodes(nodes);
Some(Overlay { parents, meta })
};
let parents_of = |set_id: &AccountSetId| -> Option<&[AccountSetId]> {
let known = snapshot.meta.contains_key(set_id)
|| overlay
.as_ref()
.is_some_and(|overlay| overlay.meta.contains_key(set_id));
known.then(|| {
snapshot
.parents
.get(set_id)
.or_else(|| {
overlay
.as_ref()
.and_then(|overlay| overlay.parents.get(set_id))
})
.map(Vec::as_slice)
.unwrap_or(&[])
})
};
match has_duplicate_account_membership_paths(members, &probe.seeds, parents_of) {
Some(false) => {
span.record(
"path",
if overlay.is_some() {
"supplement"
} else {
"memory"
},
);
Ok(())
}
Some(true) => Err(AccountSetError::MemberAlreadyAdded),
None => {
span.record("path", "fallback_unknown");
self.inner
.repo
.assert_no_double_membership(op, members)
.await
}
}
}
#[instrument(
level = "debug",
name = "account_set.assert_valid_set_memberships",
skip_all,
fields(count = members.len(), path = tracing::field::Empty),
err(level = "warn")
)]
pub(super) async fn assert_valid_set_memberships_in_op(
&self,
op: &mut impl es_entity::AtomicOperation,
members: &[SetMembership],
) -> Result<(), AccountSetError> {
let span = tracing::Span::current();
let snapshot = self.load();
let epoch = self.inner.repo.fetch_set_graph_epoch_in_op(op).await?;
let existing_edges = if snapshot.epoch == epoch {
span.record("path", "memory");
snapshot.edges_connected_to(members)
} else {
span.record(
"path",
if snapshot.epoch == COLD_EPOCH {
"fallback_cold"
} else {
"fallback_epoch"
},
);
self.spawn_refresh();
self.inner.repo.fetch_set_membership_edges_in_op(op).await?
};
let account_members = self
.inner
.repo
.fetch_affected_account_memberships_in_op(op, &existing_edges, members)
.await?;
validate_set_memberships(&existing_edges, members, &account_members)
}
fn load(&self) -> Arc<GraphSnapshot> {
self.inner
.snapshot
.read()
.expect("set_graph_cache snapshot lock poisoned")
.clone()
}
fn index_nodes(nodes: Vec<SetGraphNode>) -> IndexedNodes {
let mut parents: HashMap<AccountSetId, Vec<AccountSetId>> = HashMap::new();
let mut children: HashMap<AccountSetId, Vec<AccountSetId>> = HashMap::new();
let mut meta = HashMap::new();
for node in nodes {
meta.insert(
node.id,
SetMeta {
journal_id: node.journal_id,
eventually_consistent: node.eventually_consistent,
},
);
if let Some(parent_id) = node.parent_id {
parents.entry(node.id).or_default().push(parent_id);
children.entry(parent_id).or_default().push(node.id);
}
}
IndexedNodes {
parents,
children,
meta,
}
}
#[allow(clippy::type_complexity)]
fn expand<'c>(
snapshot: &GraphSnapshot,
overlay: Option<&Overlay>,
journal_id: JournalId,
seeds: &[AccountMembership],
(entry_account_ids, entry_currencies): &(Vec<AccountId>, Vec<&'c str>),
) -> Option<(
HashMap<AccountId, Vec<AccountSetId>>,
(Vec<AccountSetId>, Vec<&'c str>),
)> {
let meta_of = |set_id: &AccountSetId| -> Option<SetMeta> {
snapshot
.meta
.get(set_id)
.or_else(|| overlay.and_then(|o| o.meta.get(set_id)))
.copied()
};
let parents_of = |set_id: &AccountSetId| -> &[AccountSetId] {
snapshot
.parents
.get(set_id)
.or_else(|| overlay.and_then(|o| o.parents.get(set_id)))
.map(Vec::as_slice)
.unwrap_or(&[])
};
let mut per_account: HashMap<AccountId, Vec<AccountSetId>> = HashMap::new();
for seed in seeds {
per_account
.entry(seed.account_id)
.or_default()
.push(seed.account_set_id);
}
let mut mappings = HashMap::new();
let mut non_ec: HashMap<AccountId, Vec<AccountSetId>> = HashMap::new();
for (account_id, seed_sets) in per_account {
let mut visited: HashSet<AccountSetId> = HashSet::new();
let mut queue: VecDeque<AccountSetId> = seed_sets.into();
let mut ancestors = Vec::new();
let mut non_ec_ancestors = Vec::new();
while let Some(set_id) = queue.pop_front() {
if !visited.insert(set_id) {
continue;
}
let meta = meta_of(&set_id)?;
if meta.journal_id == journal_id {
ancestors.push(set_id);
if !meta.eventually_consistent {
non_ec_ancestors.push(set_id);
}
}
queue.extend(parents_of(&set_id));
}
if !non_ec_ancestors.is_empty() {
non_ec.insert(account_id, non_ec_ancestors);
}
if !ancestors.is_empty() {
mappings.insert(account_id, ancestors);
}
}
let mut lock_pairs: Vec<(AccountSetId, &str)> = entry_account_ids
.iter()
.zip(entry_currencies.iter())
.flat_map(|(account_id, currency)| {
non_ec
.get(account_id)
.into_iter()
.flatten()
.map(move |set_id| (*set_id, *currency))
})
.collect();
lock_pairs.sort_unstable();
lock_pairs.dedup();
Some((mappings, lock_pairs.into_iter().unzip()))
}
fn spawn_refresh(&self) {
let inner = Arc::clone(&self.inner);
tokio::spawn(async move {
if let Err(error) = Self::refresh(&inner).await {
tracing::warn!(%error, "set_graph_cache refresh failed");
}
});
}
#[instrument(
level = "debug",
name = "cala_ledger.set_graph_cache.refresh",
skip_all,
fields(epoch = tracing::field::Empty, sets = tracing::field::Empty),
err(level = "warn")
)]
async fn refresh(inner: &SetGraphCacheInner) -> Result<(), AccountSetError> {
let Ok(_guard) = inner.refresh_lock.try_lock() else {
return Ok(());
};
let data = inner.repo.fetch_set_graph().await?;
let IndexedNodes {
parents,
children,
meta,
} = Self::index_nodes(data.nodes);
let new = GraphSnapshot {
epoch: data.epoch,
parents,
children,
meta,
};
tracing::Span::current().record("epoch", new.epoch);
tracing::Span::current().record("sets", new.meta.len());
let mut current = inner
.snapshot
.write()
.expect("set_graph_cache snapshot lock poisoned");
if new.epoch >= current.epoch {
*current = Arc::new(new);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn edge(account_set_id: AccountSetId, member_account_set_id: AccountSetId) -> SetMembership {
SetMembership {
account_set_id,
member_account_set_id,
}
}
#[test]
fn batch_validation_selects_only_connected_snapshot_edges() {
let root = AccountSetId::new();
let branch = AccountSetId::new();
let leaf = AccountSetId::new();
let proposed_leaf = AccountSetId::new();
let unrelated_root = AccountSetId::new();
let unrelated_leaf = AccountSetId::new();
let snapshot = GraphSnapshot {
epoch: 1,
parents: HashMap::from([
(branch, vec![root]),
(leaf, vec![branch]),
(unrelated_leaf, vec![unrelated_root]),
]),
children: HashMap::from([
(root, vec![branch]),
(branch, vec![leaf]),
(unrelated_root, vec![unrelated_leaf]),
]),
meta: HashMap::new(),
};
let edges: HashSet<_> = snapshot
.edges_connected_to(&[edge(branch, proposed_leaf)])
.into_iter()
.collect();
assert_eq!(
edges,
HashSet::from([edge(root, branch), edge(branch, leaf)])
);
}
#[test]
fn batch_validation_selects_each_component_joined_by_proposed_edges() {
let left_root = AccountSetId::new();
let left_leaf = AccountSetId::new();
let right_root = AccountSetId::new();
let right_leaf = AccountSetId::new();
let unrelated_root = AccountSetId::new();
let unrelated_leaf = AccountSetId::new();
let snapshot = GraphSnapshot {
epoch: 1,
parents: HashMap::from([
(left_leaf, vec![left_root]),
(right_leaf, vec![right_root]),
(unrelated_leaf, vec![unrelated_root]),
]),
children: HashMap::from([
(left_root, vec![left_leaf]),
(right_root, vec![right_leaf]),
(unrelated_root, vec![unrelated_leaf]),
]),
meta: HashMap::new(),
};
let edges: HashSet<_> = snapshot
.edges_connected_to(&[edge(left_leaf, right_root)])
.into_iter()
.collect();
assert_eq!(
edges,
HashSet::from([edge(left_root, left_leaf), edge(right_root, right_leaf)])
);
}
}