mod error;
mod repo;
mod template_cache;
use std::collections::{HashMap, HashSet};
use chrono::{DateTime, Utc};
use es_entity::AtomicOperation;
use tracing::instrument;
use cala_types::{balance::BalanceSnapshot, entry::EntryValues};
use crate::{
account_set::AccountSets,
balance::Balances,
outbox::OutboxPublisher,
primitives::*,
transaction::Transaction,
tx_template::{Params, PreparedTransaction, TxTemplates},
velocity::Velocities,
};
pub use error::{PostingError, RejectionReason};
pub(crate) type AncestorMappings = HashMap<JournalId, HashMap<AccountId, Vec<AccountSetId>>>;
use repo::{BalanceKeys, PostingRepo, PostingRows, PostingState};
use template_cache::{ResolvedTemplate, TemplateCache};
#[derive(Debug, Clone)]
pub struct PostingInput {
pub tx_id: TransactionId,
pub tx_template_code: String,
pub params: Params,
}
impl PostingInput {
pub fn new(
tx_id: TransactionId,
tx_template_code: impl Into<String>,
params: impl Into<Params>,
) -> Self {
Self {
tx_id,
tx_template_code: tx_template_code.into(),
params: params.into(),
}
}
}
#[derive(Clone)]
pub struct Postings {
repo: PostingRepo,
tx_templates: TxTemplates,
account_sets: AccountSets,
balances: Balances,
velocities: Velocities,
publisher: OutboxPublisher,
templates: TemplateCache,
}
impl Postings {
pub(crate) fn new(
publisher: &OutboxPublisher,
tx_templates: &TxTemplates,
account_sets: &AccountSets,
balances: &Balances,
velocities: &Velocities,
) -> Self {
let repo = PostingRepo;
Self {
templates: TemplateCache::new(repo.clone()),
repo,
tx_templates: tx_templates.clone(),
account_sets: account_sets.clone(),
balances: balances.clone(),
velocities: velocities.clone(),
publisher: publisher.clone(),
}
}
#[instrument(
level = "debug",
name = "cala_ledger.posting.post_all_in_op",
skip_all,
fields(
batch_size = batch.len(),
failed_posting_index = tracing::field::Empty,
failed_posting_id = tracing::field::Empty,
),
err(level = "warn")
)]
pub(crate) async fn post_all_in_op(
&self,
db: &mut impl AtomicOperation,
batch: Vec<PostingInput>,
) -> Result<Vec<Transaction>, PostingError> {
if batch.is_empty() {
return Ok(Vec::new());
}
let codes: Vec<String> = Self::dedup(batch.iter().map(|p| p.tx_template_code.clone()));
let used = self.templates.resolve_in_op(db, &codes).await?;
let mut prepared = self.prepare_all(&batch, &used)?;
let mut keys = Self::entry_balance_keys(&prepared);
if keys.account_ids.len() > error::MAX_DISTINCT_BALANCES_PER_BATCH {
return Err(PostingError::BatchTooManyAccounts {
distinct: keys.account_ids.len(),
max: error::MAX_DISTINCT_BALANCES_PER_BATCH,
});
}
let locked = self
.repo
.lock_balances_and_probe_templates_in_op(db, &keys, &codes, db.maybe_now())
.await?;
if let Err(stale) = TemplateCache::assert_up_to_date(&used, &locked.template_versions) {
let refreshed = self.templates.refresh_in_op(db, &stale).await?;
let mut merged = used;
merged.extend(refreshed);
prepared = self.prepare_all(&batch, &merged)?;
let new_keys = Self::entry_balance_keys(&prepared);
self.repo
.lock_balances_and_probe_templates_in_op(db, &new_keys, &[], db.maybe_now())
.await?;
keys = new_keys;
}
let now = locked.now;
let account_ids = Self::dedup(
prepared
.iter()
.flat_map(|p| p.entries.iter().map(|e| e.account_id())),
);
let journal_ids = Self::dedup(prepared.iter().map(|p| p.journal_id));
let mut read = self
.repo
.read_posting_state_in_op(db, &account_ids, &journal_ids, &keys)
.await?;
self.validate(&batch, &prepared, &read)?;
let mappings = self.resolve_ancestors(db, &prepared, &mut read).await?;
let (transactions, entries_per_posting) = prepared
.into_iter()
.map(|p| p.into_new_transaction(now))
.collect::<(Vec<_>, Vec<_>)>();
let mut hydrated = Vec::with_capacity(transactions.len());
let mut entry_values: Vec<Vec<EntryValues>> = Vec::with_capacity(transactions.len());
let mut rows = PostingRows::default();
for (new_tx, new_entries) in transactions.into_iter().zip(entries_per_posting) {
let transaction = rows.push_transaction(new_tx, now);
let mut values = Vec::with_capacity(new_entries.len());
for new_entry in new_entries {
let entry = rows.push_entry(new_entry, now);
values.push(entry.into_values());
}
entry_values.push(values);
hydrated.push(transaction);
}
let snapshots = self.fold_balances(&hydrated, &entry_values, &read, &mappings, now);
let for_enforcement: Vec<(&cala_types::transaction::TransactionValues, &[EntryValues])> =
hydrated
.iter()
.zip(entry_values.iter())
.map(|(tx, values)| (tx.values(), values.as_slice()))
.collect();
self.velocities
.enforce_batch_in_op(db, now, &for_enforcement, &read.controls, &mappings)
.await?;
self.repo
.insert_postings_and_balances_in_op(db, now, &rows, &snapshots)
.await?;
self.update_effective_balances(db, &hydrated, &entry_values, &read, &mappings, now)
.await?;
let mut payloads = Vec::new();
for (transaction, values) in hydrated.iter().zip(entry_values.iter()) {
payloads.push(crate::outbox::OutboxEventPayload::TransactionCreated {
transaction: transaction.values().clone(),
});
payloads.extend(values.iter().map(|entry| {
crate::outbox::OutboxEventPayload::EntryCreated {
entry: entry.clone(),
}
}));
}
self.publisher.publish_all(db, payloads.into_iter()).await?;
Ok(hydrated)
}
fn prepare_all(
&self,
batch: &[PostingInput],
templates: &HashMap<String, ResolvedTemplate>,
) -> Result<Vec<PreparedTransaction>, PostingError> {
let mut prepared = Vec::with_capacity(batch.len());
let mut seen_ids = HashSet::new();
let mut seen_external = HashSet::new();
for (index, input) in batch.iter().enumerate() {
let template = templates
.get(&input.tx_template_code)
.expect("template resolved above");
let posting = self
.tx_templates
.prepare_transaction(input.tx_id, &template.values, input.params.clone())
.map_err(|e| PostingError::rejected(index, input.tx_id, e))?;
if !seen_ids.insert(posting.tx_id) {
return Err(PostingError::rejected(
index,
input.tx_id,
RejectionReason::DuplicateTransactionIdInBatch(posting.tx_id),
));
}
if let Some(external_id) = posting.external_id.as_ref() {
if !seen_external.insert(external_id.clone()) {
return Err(PostingError::rejected(
index,
input.tx_id,
RejectionReason::DuplicateExternalIdInBatch(external_id.clone()),
));
}
}
prepared.push(posting);
}
Ok(prepared)
}
fn validate(
&self,
batch: &[PostingInput],
prepared: &[PreparedTransaction],
read: &PostingState,
) -> Result<(), PostingError> {
for (index, posting) in prepared.iter().enumerate() {
let tx_id = batch[index].tx_id;
match read.journals.get(&posting.journal_id) {
None => {
return Err(PostingError::rejected(
index,
tx_id,
RejectionReason::JournalNotFound(posting.journal_id),
))
}
Some(journal) if journal.status == Status::Locked => {
return Err(PostingError::rejected(
index,
tx_id,
RejectionReason::JournalLocked(posting.journal_id),
))
}
Some(_) => {}
}
for entry in posting.entries.iter() {
let account_id = entry.account_id();
let Some(meta) = read.accounts.get(&account_id) else {
return Err(PostingError::rejected(
index,
tx_id,
RejectionReason::AccountNotFound(account_id),
));
};
if meta.is_account_set {
return Err(PostingError::rejected(
index,
tx_id,
RejectionReason::EntryTargetsAccountSet(account_id),
));
}
if meta.locked {
return Err(PostingError::rejected(
index,
tx_id,
RejectionReason::AccountLocked(account_id),
));
}
}
}
Ok(())
}
async fn resolve_ancestors(
&self,
db: &mut impl AtomicOperation,
prepared: &[PreparedTransaction],
read: &mut PostingState,
) -> Result<AncestorMappings, PostingError> {
let mut mappings: AncestorMappings = HashMap::new();
if read.seeds.is_empty() {
return Ok(mappings);
}
let mut journals: Vec<JournalId> = Self::dedup(prepared.iter().map(|p| p.journal_id));
journals.sort_unstable();
let mut ancestor_keys = BalanceKeys::default();
let mut ancestor_ids: Vec<AccountId> = Vec::new();
for journal_id in journals {
let entry_pairs: (Vec<AccountId>, Vec<&str>) = prepared
.iter()
.filter(|p| p.journal_id == journal_id)
.flat_map(|p| p.entries.iter())
.map(|e| (e.account_id(), e.currency().code()))
.collect::<HashSet<_>>()
.into_iter()
.unzip();
if entry_pairs.0.is_empty() {
continue;
}
let resolved = self
.account_sets
.resolve_mappings_from_probe_in_op(
db,
journal_id,
read.epoch,
&read.seeds,
&entry_pairs,
)
.await?;
for posting in prepared.iter().filter(|p| p.journal_id == journal_id) {
for entry in posting.entries.iter() {
for set_id in resolved.get(&entry.account_id()).into_iter().flatten() {
let account_id = AccountId::from(set_id);
ancestor_keys.push(journal_id, account_id, entry.currency());
ancestor_ids.push(account_id);
}
}
}
let per_journal = mappings.entry(journal_id).or_default();
for (account_id, sets) in resolved {
per_journal.entry(account_id).or_default().extend(sets);
}
}
if ancestor_keys.is_empty() {
return Ok(mappings);
}
ancestor_ids.sort_unstable();
ancestor_ids.dedup();
let supplemental = self
.repo
.read_ancestor_state_in_op(db, &ancestor_ids, &ancestor_keys.sorted_deduped())
.await?;
read.accounts.extend(supplemental.accounts);
read.balances.extend(supplemental.balances);
read.controls.extend(supplemental.controls);
for account_id in ancestor_ids {
if let Some(meta) = read.accounts.get(&account_id) {
if meta.locked {
return Err(PostingError::BalanceError(
crate::balance::error::BalanceError::AccountLocked(account_id),
));
}
}
}
Ok(mappings)
}
fn fold_balances(
&self,
transactions: &[Transaction],
entry_values: &[Vec<EntryValues>],
read: &PostingState,
mappings: &AncestorMappings,
now: DateTime<Utc>,
) -> Vec<BalanceSnapshot> {
let mut journals: Vec<JournalId> =
Self::dedup(transactions.iter().map(|tx| tx.values().journal_id));
journals.sort_unstable();
let empty = HashMap::new();
let mut all = Vec::new();
for journal_id in journals {
let mappings = mappings.get(&journal_id).unwrap_or(&empty);
let entries: Vec<EntryValues> = transactions
.iter()
.zip(entry_values)
.filter(|(tx, _)| tx.values().journal_id == journal_id)
.flat_map(|(_, values)| values.iter().cloned())
.collect();
if entries.is_empty() {
continue;
}
let mut current: HashMap<(AccountId, Currency), Option<BalanceSnapshot>> =
HashMap::new();
for entry in entries.iter() {
for account_id in mappings
.get(&entry.account_id)
.into_iter()
.flatten()
.map(AccountId::from)
.chain(std::iter::once(entry.account_id))
{
let involved = read
.accounts
.get(&account_id)
.is_some_and(|meta| !meta.eventually_consistent);
if !involved {
continue;
}
current
.entry((account_id, entry.currency))
.or_insert_with(|| {
read.balances
.get(&(journal_id, account_id, entry.currency))
.cloned()
});
}
}
all.extend(crate::balance::Snapshots::from_entries(
now, current, &entries, mappings,
));
}
all
}
async fn update_effective_balances(
&self,
db: &mut impl AtomicOperation,
transactions: &[Transaction],
entry_values: &[Vec<EntryValues>],
read: &PostingState,
mappings: &AncestorMappings,
now: DateTime<Utc>,
) -> Result<(), PostingError> {
let mut groups: Vec<((JournalId, chrono::NaiveDate), Vec<EntryValues>)> = Vec::new();
for (transaction, entries) in transactions.iter().zip(entry_values) {
let journal_id = transaction.values().journal_id;
let enabled = read
.journals
.get(&journal_id)
.is_some_and(|j| j.config.enable_effective_balances);
if !enabled {
continue;
}
let key = (journal_id, transaction.values().effective);
match groups.iter_mut().find(|(k, _)| *k == key) {
Some((_, group)) => group.extend(entries.iter().cloned()),
None => groups.push((key, entries.clone())),
}
}
groups.sort_by_key(|((journal_id, effective), _)| (*journal_id, *effective));
let empty = HashMap::new();
for ((journal_id, effective), entries) in groups {
let mappings = mappings.get(&journal_id).unwrap_or(&empty);
let involved: (Vec<AccountId>, Vec<&str>) = entries
.iter()
.flat_map(|entry| {
mappings
.get(&entry.account_id)
.into_iter()
.flatten()
.map(AccountId::from)
.chain(std::iter::once(entry.account_id))
.map(move |id| (id, entry.currency))
})
.filter(|(id, _)| {
read.accounts
.get(id)
.is_some_and(|meta| !meta.eventually_consistent)
})
.collect::<HashSet<_>>()
.into_iter()
.map(|(id, currency)| (id, currency.code()))
.unzip();
if involved.0.is_empty() {
continue;
}
self.balances
.effective()
.update_cumulative_balances_in_op(
db,
journal_id,
entries,
effective,
now,
mappings.clone(),
involved,
)
.await?;
}
Ok(())
}
fn dedup<T: Ord + Clone>(items: impl Iterator<Item = T>) -> Vec<T> {
let mut out: Vec<T> = items.collect();
out.sort_unstable();
out.dedup();
out
}
fn entry_balance_keys(prepared: &[PreparedTransaction]) -> BalanceKeys {
let mut keys = BalanceKeys::default();
for posting in prepared {
for entry in posting.entries.iter() {
keys.push(posting.journal_id, entry.account_id(), entry.currency());
}
}
keys.sorted_deduped()
}
}