use super::log::Checkpoint;
use super::stamp_proof::StampProof;
use super::store::write_tx;
use super::{Shelf, ShelfError, Time};
use cll::mmr;
use rusqlite::{params, Connection, OptionalExtension, Transaction};
use std::collections::btree_map::Entry;
use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
pub trait ReceiveProfile: Send + Sync {
fn read(&self, record: &[u8]) -> Result<ReceivedRecord, String>;
fn admit(&self, record: &ReceivedRecord) -> Result<bool, String>;
fn answered(&self, offer: &ReceivedRecord) -> Result<bool, String>;
fn proof_sender(&self, checkpoint_key_id: &str, log_id: &str)
-> Result<Option<String>, String>;
fn after_receive(&self, result: &ReceiveResult, at: Time) -> Result<(), String> {
let _ = (result, at);
Ok(())
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ReceivedRecord {
pub digest: String,
pub sender_key: String,
pub log_id: String,
pub position: u64,
pub prev: String,
pub deal_id: String,
pub offer: bool,
}
#[derive(Clone, Default)]
pub struct ReceiveConfig {
pub profile: Option<Arc<dyn ReceiveProfile>>,
pub bounds: ReceiveBounds,
pub retention: Duration,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ReceiveBounds {
pub max_record_bytes: u64,
pub max_records_per_log: u64,
pub max_offers_per_sender: u64,
pub max_offer_bytes_per_sender: u64,
pub max_offers: u64,
pub max_offer_bytes: u64,
pub max_offer_evidence_per_sender: u64,
pub max_offer_evidence: u64,
}
impl ReceiveBounds {
pub const DEFAULT: ReceiveBounds = ReceiveBounds {
max_record_bytes: 1 << 20,
max_records_per_log: 100_000,
max_offers_per_sender: 16,
max_offer_bytes_per_sender: 4 << 20,
max_offers: 1024,
max_offer_bytes: 64 << 20,
max_offer_evidence_per_sender: 16,
max_offer_evidence: 1024,
};
pub fn with_defaults(self) -> Self {
let d = Self::DEFAULT;
let or = |v: u64, def: u64| if v == 0 { def } else { v };
Self {
max_record_bytes: or(self.max_record_bytes, d.max_record_bytes),
max_records_per_log: or(self.max_records_per_log, d.max_records_per_log),
max_offers_per_sender: or(self.max_offers_per_sender, d.max_offers_per_sender),
max_offer_bytes_per_sender: or(
self.max_offer_bytes_per_sender,
d.max_offer_bytes_per_sender,
),
max_offers: or(self.max_offers, d.max_offers),
max_offer_bytes: or(self.max_offer_bytes, d.max_offer_bytes),
max_offer_evidence_per_sender: or(
self.max_offer_evidence_per_sender,
d.max_offer_evidence_per_sender,
),
max_offer_evidence: or(self.max_offer_evidence, d.max_offer_evidence),
}
}
}
pub const EVIDENCE_FORK: &str = "fork";
pub const EVIDENCE_CHECKPOINT: &str = "checkpoint";
pub const EVIDENCE_CHECKPOINT_RECORDS: &str = "checkpoint_records";
#[derive(Debug, Default)]
pub struct ReceiveResult {
pub record: Option<RecordOutcome>,
pub proof: Option<ProofOutcome>,
pub changes: ReceiveChanges,
}
#[derive(Debug, Default)]
pub struct RecordOutcome {
pub digest: String,
pub deal_id: String,
pub stored: bool,
pub duplicate: bool,
pub deleted: bool,
pub counted: bool,
pub ask_for_book: bool,
pub refused: Option<ShelfError>,
}
#[derive(Debug, Default)]
pub struct ProofOutcome {
pub accepted: bool,
pub duplicate: bool,
pub refused: Option<ShelfError>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ReceiveChanges {
pub grew: Vec<String>,
pub shrank: Vec<String>,
pub forked: Vec<String>,
pub covered: Vec<String>,
}
#[derive(Debug)]
pub struct ReceiveError {
pub result: ReceiveResult,
pub error: Option<ShelfError>,
pub after: Option<ShelfError>,
}
impl ReceiveError {
fn of(error: ShelfError) -> Box<Self> {
Box::new(Self {
result: ReceiveResult::default(),
error: Some(error),
after: None,
})
}
pub fn errors(&self) -> Vec<&ShelfError> {
let after = self.after.as_ref();
if let Some(e) = &self.error {
return std::iter::once(e).chain(after).collect();
}
let record = self.result.record.as_ref().and_then(|r| r.refused.as_ref());
let proof = self.result.proof.as_ref().and_then(|p| p.refused.as_ref());
record.into_iter().chain(proof).chain(after).collect()
}
}
impl fmt::Display for ReceiveError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let parts: Vec<String> = self.errors().iter().map(|e| e.to_string()).collect();
f.write_str(&parts.join("\n"))
}
}
impl std::error::Error for ReceiveError {}
#[derive(Default)]
pub(super) struct ChangeSet {
grew: BTreeSet<String>,
shrank: BTreeSet<String>,
pub(super) forked: BTreeSet<String>,
covered: BTreeSet<String>,
}
impl ChangeSet {
pub(super) fn merge(&mut self, o: ChangeSet) {
self.grew.extend(o.grew);
self.shrank.extend(o.shrank);
self.forked.extend(o.forked);
self.covered.extend(o.covered);
}
fn result(&self) -> ReceiveChanges {
let list = |s: &BTreeSet<String>| s.iter().cloned().collect();
ReceiveChanges {
grew: list(&self.grew),
shrank: list(&self.shrank),
forked: list(&self.forked),
covered: list(&self.covered),
}
}
}
pub(super) fn received_time(t: Time) -> String {
let date = t.rfc3339();
let nanos = t.unix_nanos().rem_euclid(1_000_000_000);
format!("{}.{nanos:09}Z", &date[..date.len() - 1])
}
pub(super) fn parse_received_time(text: &str, what: &str) -> Result<Time, ShelfError> {
Time::parse(text).ok_or_else(|| ShelfError::Corrupt(format!("{what}'s time {text:?}")))
}
pub(super) fn sql_int(v: u64) -> Result<i64, ShelfError> {
i64::try_from(v).map_err(|_| ShelfError::Invalid(format!("{v} is past 2^63 - 1")))
}
pub(super) fn unsigned(r: &rusqlite::Row<'_>, i: usize) -> rusqlite::Result<u64> {
let v: i64 = r.get(i)?;
u64::try_from(v).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(i, v))
}
fn is_digest(s: &str) -> bool {
s.len() == 64 && s.bytes().all(|c| matches!(c, b'0'..=b'9' | b'a'..=b'f'))
}
pub(super) type LogKey = (String, String);
pub(super) enum TxError {
Stale,
Refusal(ShelfError),
Store(ShelfError),
}
impl From<ShelfError> for TxError {
fn from(e: ShelfError) -> Self {
TxError::Store(e)
}
}
impl From<rusqlite::Error> for TxError {
fn from(e: rusqlite::Error) -> Self {
TxError::Store(e.into())
}
}
#[cfg(test)]
pub(super) type CommitHook = Box<dyn Fn(&str) -> bool>;
#[cfg(test)]
thread_local! {
pub(super) static BEFORE_COMMIT: std::cell::RefCell<Option<CommitHook>> =
const { std::cell::RefCell::new(None) };
}
pub(super) fn before_commit(_part: &str) -> Result<(), TxError> {
#[cfg(test)]
{
let fail = BEFORE_COMMIT.with(|h| h.borrow().as_ref().is_some_and(|f| f(_part)));
if fail {
return Err(TxError::Store(ShelfError::Invalid(
"the commit was interrupted".into(),
)));
}
}
Ok(())
}
pub(super) struct LogState {
pub(super) prefix: PrefixState,
coverage: u64,
}
pub(super) fn read_log_state(conn: &Connection, k: &LogKey) -> Result<LogState, ShelfError> {
let prefix = received_prefix(conn, k)?;
let coverage = verified_coverage(conn, k, &prefix)?;
Ok(LogState { prefix, coverage })
}
pub(super) fn report(before: &LogState, after: &LogState, changes: &mut ChangeSet) {
let was: BTreeSet<&str> = before
.prefix
.counted
.iter()
.map(|c| c.digest.as_str())
.collect();
let now: BTreeSet<&str> = after
.prefix
.counted
.iter()
.map(|c| c.digest.as_str())
.collect();
for c in &after.prefix.counted {
if !was.contains(c.digest.as_str()) {
changes.grew.insert(c.deal.clone());
}
}
for c in &before.prefix.counted {
if !now.contains(c.digest.as_str()) {
changes.shrank.insert(c.deal.clone());
}
}
for i in before.coverage..after.coverage {
changes
.covered
.insert(after.prefix.counted[i as usize].deal.clone());
}
}
const OFFER_ATTEMPTS: u32 = 3;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct OwnMark {
records: i64,
entries: i64,
}
fn read_own_mark(conn: &Connection) -> Result<OwnMark, ShelfError> {
let (records, logs): (i64, i64) = conn.query_row(
"SELECT (SELECT COUNT(*) FROM book_records), (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'cll_entries')",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
let mut m = OwnMark {
records,
entries: 0,
};
if logs > 0 {
m.entries = conn.query_row(
"SELECT COUNT(*) FROM cll_entries WHERE log_id IN (SELECT log_id FROM shelf_logs WHERE kind IN ('book', 'journal'))",
[],
|r| r.get(0),
)?;
}
Ok(m)
}
#[derive(Clone, Debug)]
pub(super) struct ReceivedRow {
digest: String,
sender: String,
log: String,
position: u64,
prev: String,
deal: String,
offer: bool,
}
impl ReceivedRow {
pub(super) fn record(&self) -> ReceivedRecord {
ReceivedRecord {
digest: self.digest.clone(),
sender_key: self.sender.clone(),
log_id: self.log.clone(),
position: self.position,
prev: self.prev.clone(),
deal_id: self.deal.clone(),
offer: self.offer,
}
}
pub(super) fn from_row(r: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
Ok(Self {
digest: r.get(0)?,
sender: r.get(1)?,
log: r.get(2)?,
position: unsigned(r, 3)?,
prev: r.get(4)?,
deal: r.get(5)?,
offer: r.get(6)?,
})
}
}
const RECEIVED_COLUMNS: &str = "digest, sender_key, log_id, position, prev, deal_id, offer";
#[derive(Default)]
struct OfferPlan {
answered: Vec<String>,
drop: Vec<ReceivedRow>,
}
#[derive(Clone, Debug)]
pub(super) struct CountedRecord {
digest: String,
deal: String,
}
#[derive(Clone, Debug, Default)]
pub(super) struct PrefixState {
pub(super) counted: Vec<CountedRecord>,
fork_at: u64,
pub(super) deals: Vec<String>,
}
impl PrefixState {
fn counts(&self, position: u64, digest: &str) -> bool {
position >= 1
&& position <= self.counted.len() as u64
&& self.counted[(position - 1) as usize].digest == digest
}
pub(super) fn digests(&self, n: u64) -> Vec<String> {
self.counted[..n as usize]
.iter()
.map(|c| c.digest.clone())
.collect()
}
}
pub(super) fn received_prefix(conn: &Connection, k: &LogKey) -> Result<PrefixState, ShelfError> {
let held: Vec<ReceivedRow> = {
let mut stmt = conn.prepare(&format!(
"SELECT {RECEIVED_COLUMNS} FROM shelf_received WHERE sender_key = ? AND log_id = ? ORDER BY position, digest"
))?;
let rows = stmt.query_map(params![k.0, k.1], ReceivedRow::from_row)?;
rows.collect::<Result<_, _>>()?
};
let recorded: u64 = conn.query_row(
"SELECT COALESCE(MIN(at), 0) FROM shelf_received_conflicts WHERE sender_key = ? AND log_id = ? AND kind = ?",
params![k.0, k.1, EVIDENCE_FORK],
|r| unsigned(r, 0),
)?;
let mut out = PrefixState {
fork_at: recorded,
..Default::default()
};
let mut at: BTreeMap<u64, Vec<&ReceivedRow>> = BTreeMap::new();
let mut claims: BTreeMap<u64, BTreeSet<&str>> = BTreeMap::new();
let mut deals = BTreeSet::new();
for r in &held {
at.entry(r.position).or_default().push(r);
claims.entry(r.position).or_default().insert(&r.digest);
if r.position > 1 {
claims.entry(r.position - 1).or_default().insert(&r.prev);
}
deals.insert(r.deal.clone());
}
for (p, c) in &claims {
if c.len() > 1 && (out.fork_at == 0 || *p < out.fork_at) {
out.fork_at = *p;
}
}
let mut p = 1;
while at.get(&p).is_some_and(|rows| rows.len() == 1) && (out.fork_at == 0 || p < out.fork_at) {
let r = at[&p][0];
out.counted.push(CountedRecord {
digest: r.digest.clone(),
deal: r.deal.clone(),
});
p += 1;
}
out.deals = deals.into_iter().collect();
Ok(out)
}
struct HeldFork {
position: u64,
held: Vec<u8>,
deal: String,
}
fn fork_evidence(tx: &Transaction<'_>, r: &ReceivedRecord) -> Result<Vec<HeldFork>, ShelfError> {
let mut checks = vec![(r.position, r.digest.as_str())];
if r.position > 1 {
checks.push((r.position - 1, r.prev.as_str()));
}
let mut out = Vec::new();
for (position, digest) in checks {
let next = position
.checked_add(1)
.and_then(|n| i64::try_from(n).ok())
.unwrap_or(-1);
type Row = (String, String, i64, String, Vec<u8>);
let rows: Vec<Row> = {
let mut stmt = tx.prepare(
"SELECT digest, prev, position, deal_id, record FROM shelf_received WHERE sender_key = ? AND log_id = ? AND position IN (?, ?) ORDER BY position, digest",
)?;
let rows = stmt.query_map(
params![r.sender_key, r.log_id, sql_int(position)?, next],
|row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
},
)?;
rows.collect::<Result<_, _>>()?
};
let mut found = rows.into_iter().find_map(|(d, prev, at, deal, b)| {
let claimed = if at == next { prev } else { d };
(claimed != digest).then_some(HeldFork {
position,
held: b,
deal,
})
});
if found.is_none() {
found = tx
.query_row(
"SELECT digest, deal_id FROM shelf_received_vacated WHERE sender_key = ? AND log_id = ? AND position = ? AND digest != ? ORDER BY digest LIMIT 1",
params![r.sender_key, r.log_id, sql_int(position)?, digest],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
)
.optional()?
.map(|(d, deal)| HeldFork {
position,
held: d.into_bytes(),
deal,
});
}
out.extend(found);
}
Ok(out)
}
fn offer_room(tx: &Transaction<'_>, sender: &str, b: &ReceiveBounds) -> Result<bool, ShelfError> {
let (mine, all): (i64, i64) = tx.query_row(
"SELECT COALESCE(SUM(sender_key = ?), 0), COUNT(*) FROM (
SELECT sender_key FROM shelf_received_vacated WHERE offer = 1
UNION ALL SELECT sender_key FROM shelf_received_conflicts WHERE offer = 1)",
[sender],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
Ok((mine as u64) < b.max_offer_evidence_per_sender && (all as u64) < b.max_offer_evidence)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn note_conflict(
tx: &Transaction<'_>,
k: &LogKey,
kind: &str,
at: u64,
a: &[u8],
b: &[u8],
now: &str,
offers: bool,
bounds: &ReceiveBounds,
) -> Result<bool, ShelfError> {
let n: i64 = tx.query_row(
"SELECT COUNT(*) FROM shelf_received_conflicts WHERE sender_key = ? AND log_id = ? AND kind = ? AND at = ?",
params![k.0, k.1, kind, sql_int(at)?],
|r| r.get(0),
)?;
if n > 0 {
return Ok(false);
}
if offers && !offer_room(tx, &k.0, bounds)? {
return Ok(true);
}
tx.execute(
"INSERT INTO shelf_evidence (kind, a, b, noted_at) VALUES (?, ?, ?, ?)",
params![kind, a, b, now],
)?;
let id = tx.last_insert_rowid();
tx.execute(
"INSERT INTO shelf_received_conflicts (sender_key, log_id, kind, at, evidence_id, offer) VALUES (?, ?, ?, ?, ?, ?)",
params![k.0, k.1, kind, sql_int(at)?, id, offers],
)?;
Ok(true)
}
pub(super) const PROOF_UNCHECKED: i64 = 0;
pub(super) const PROOF_RECOMPUTED: i64 = 1;
pub(super) const PROOF_CONTRADICTED: i64 = 2;
fn vacate(
tx: &Transaction<'_>,
rows: &[ReceivedRow],
offers: bool,
bounds: &ReceiveBounds,
) -> Result<(), ShelfError> {
let mut lowest: BTreeMap<LogKey, u64> = BTreeMap::new();
for r in rows {
let keep = !offers || offer_room(tx, &r.sender, bounds)?;
if keep {
tx.execute(
"INSERT OR IGNORE INTO shelf_received_vacated (sender_key, log_id, position, digest, deal_id, offer) VALUES (?, ?, ?, ?, ?, ?)",
params![r.sender, r.log, sql_int(r.position)?, r.digest, r.deal, offers],
)?;
}
let low = lowest
.entry((r.sender.clone(), r.log.clone()))
.or_insert(r.position);
*low = (*low).min(r.position);
}
for (k, position) in lowest {
let sizes: Vec<u64> = {
let mut stmt = tx.prepare(
"SELECT DISTINCT size FROM shelf_received_proofs WHERE sender_key = ? AND log_id = ? AND verified = ?",
)?;
let rows = stmt.query_map(params![k.0, k.1, PROOF_RECOMPUTED], |r| unsigned(r, 0))?;
rows.collect::<Result<_, _>>()?
};
for size in sizes {
let leaves = mmr::leaf_count(size).unwrap_or(0);
if leaves >= position {
tx.execute(
"UPDATE shelf_received_proofs SET verified = ? WHERE sender_key = ? AND log_id = ? AND size = ?",
params![PROOF_UNCHECKED, k.0, k.1, sql_int(size)?],
)?;
}
}
}
Ok(())
}
fn check_held_proofs(
tx: &Transaction<'_>,
k: &LogKey,
p: &PrefixState,
now: &str,
changes: &mut ChangeSet,
) -> Result<(), ShelfError> {
let proofs: Vec<(u64, Vec<u8>)> = {
let mut stmt = tx.prepare(
"SELECT DISTINCT size, book_checkpoint FROM shelf_received_proofs WHERE sender_key = ? AND log_id = ? AND verified = ? ORDER BY size",
)?;
let rows = stmt.query_map(params![k.0, k.1, PROOF_UNCHECKED], |r| {
Ok((unsigned(r, 0)?, r.get(1)?))
})?;
rows.collect::<Result<_, _>>()?
};
for (size, statement) in proofs {
let cp = Checkpoint::from_statement(&statement)?;
if cp.entries > p.counted.len() as u64 {
continue;
}
let (matched, over) = root_recomputes(&cp, &p.digests(cp.entries))?;
let mut state = PROOF_RECOMPUTED;
if !matched {
state = PROOF_CONTRADICTED;
let fresh = note_conflict(
tx,
k,
EVIDENCE_CHECKPOINT_RECORDS,
size,
&statement,
&over,
now,
false,
&ReceiveBounds::default(),
)?;
if fresh {
changes.forked.extend(p.deals.iter().cloned());
}
}
tx.execute(
"UPDATE shelf_received_proofs SET verified = ? WHERE sender_key = ? AND log_id = ? AND size = ?",
params![state, k.0, k.1, sql_int(size)?],
)?;
}
Ok(())
}
fn verified_coverage(conn: &Connection, k: &LogKey, p: &PrefixState) -> Result<u64, ShelfError> {
let size: u64 = conn.query_row(
"SELECT COALESCE(MAX(size), 0) FROM shelf_received_proofs WHERE sender_key = ? AND log_id = ? AND verified = ?",
params![k.0, k.1, PROOF_RECOMPUTED],
|r| unsigned(r, 0),
)?;
let leaves = mmr::leaf_count(size).map_err(|_| {
ShelfError::Corrupt(format!("a held proof's size {size} is not a complete MMR"))
})?;
Ok(leaves.min(p.counted.len() as u64))
}
pub(super) fn root_recomputes(
cp: &Checkpoint,
digests: &[String],
) -> Result<(bool, Vec<u8>), ShelfError> {
let encoded = serde_json::to_vec(digests).map_err(|e| ShelfError::Corrupt(e.to_string()))?;
let mut tree = mmr::MemoryNodeStore::new();
for d in digests {
let value: mmr::Hash = hex::decode(d)
.ok()
.and_then(|b| b.try_into().ok())
.ok_or_else(|| ShelfError::Corrupt(format!("a held record's digest {d:?}")))?;
mmr::add_leaf(&mut tree, mmr::leaf_hash(&value))
.map_err(|e| ShelfError::Corrupt(e.to_string()))?;
}
let peaks = if cp.tree_size == 0 {
Vec::new()
} else {
if cp.tree_size > mmr::NodeReader::size(&tree) {
return Err(ShelfError::Corrupt(format!(
"invalid historical MMR size {}",
cp.tree_size
)));
}
mmr::peaks(cp.tree_size)
.map_err(|e| ShelfError::Corrupt(e.to_string()))?
.into_iter()
.map(|p| mmr::NodeReader::node(&tree, p))
.collect()
};
Ok((
hex::encode(mmr::root_from_peaks(&peaks)) == cp.root,
encoded,
))
}
fn tombstoned(conn: &Connection, digest: &str) -> Result<bool, ShelfError> {
let n: i64 = conn.query_row(
"SELECT COUNT(*) FROM shelf_received_tombstones WHERE digest = ?",
[digest],
|r| r.get(0),
)?;
Ok(n > 0)
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CountedPrefix {
pub size: u64,
pub last: String,
pub fork_at: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HeldRecord {
pub received: ReceivedRecord,
pub record: Vec<u8>,
pub received_at: Time,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Evidence {
pub kind: String,
pub a: Vec<u8>,
pub b: Vec<u8>,
pub noted_at: Time,
}
impl Shelf {
pub(super) fn live(&self) -> Result<(), ShelfError> {
if self.shared().closed.load(Ordering::SeqCst) {
return Err(ShelfError::Released);
}
Ok(())
}
pub(super) fn receive_now(&self) -> String {
received_time(self.shared().now())
}
pub(super) fn receive_tx<T>(
&self,
f: impl FnOnce(&Transaction<'_>) -> Result<T, TxError>,
) -> Result<T, TxError> {
let mut db = self.shared().db();
let tx = write_tx(&mut db)?;
let v = f(&tx)?;
tx.commit()?;
Ok(v)
}
pub fn receive(
&self,
record: Option<&[u8]>,
proof: Option<&StampProof>,
) -> Result<ReceiveResult, Box<ReceiveError>> {
let Some(profile) = self.shared().receive.profile.clone() else {
return Err(ReceiveError::of(ShelfError::Invalid(
"the shelf has no receive profile".into(),
)));
};
if record.is_none() && proof.is_none() {
return Err(ReceiveError::of(ShelfError::Invalid(
"nothing to receive".into(),
)));
}
let _serial = self
.shared()
.receive_mu
.lock()
.unwrap_or_else(|p| p.into_inner());
self.live().map_err(ReceiveError::of)?;
let mut changes = ChangeSet::default();
let mut result = ReceiveResult::default();
if let Some(record) = record {
let mut part = ChangeSet::default();
let out = self
.receive_record(profile.as_ref(), record, &mut part)
.map_err(ReceiveError::of)?;
changes.merge(part);
result.record = Some(out);
}
if let Some(proof) = proof {
let mut part = ChangeSet::default();
match self.receive_proof(profile.as_ref(), proof, &mut part) {
Ok(out) => {
changes.merge(part);
result.proof = Some(out);
}
Err(e) => {
result.changes = changes.result();
let after = self.after_receive(profile.as_ref(), &result);
return Err(Box::new(ReceiveError {
result,
error: Some(e),
after,
}));
}
}
}
result.changes = changes.result();
let after = self.after_receive(profile.as_ref(), &result);
let refused = result.record.as_ref().is_some_and(|r| r.refused.is_some())
|| result.proof.as_ref().is_some_and(|p| p.refused.is_some());
if refused || after.is_some() {
return Err(Box::new(ReceiveError {
result,
error: None,
after,
}));
}
Ok(result)
}
fn after_receive(
&self,
profile: &dyn ReceiveProfile,
result: &ReceiveResult,
) -> Option<ShelfError> {
profile
.after_receive(result, self.shared().now())
.err()
.map(|e| ShelfError::Profile(format!("after receive: {e}")))
}
fn receive_record(
&self,
profile: &dyn ReceiveProfile,
record: &[u8],
changes: &mut ChangeSet,
) -> Result<RecordOutcome, ShelfError> {
let bounds = self.shared().receive.bounds.with_defaults();
let mut out = RecordOutcome::default();
fn refuse(mut out: RecordOutcome, e: ShelfError) -> Result<RecordOutcome, ShelfError> {
out.refused = Some(e);
Ok(out)
}
if record.len() as u64 > bounds.max_record_bytes {
return refuse(
out,
ShelfError::OverLimit(format!(
"a received record of {} bytes, over the store's {}",
record.len(),
bounds.max_record_bytes
)),
);
}
let r = match profile.read(record) {
Ok(r) => r,
Err(e) => return refuse(out, ShelfError::Invalid(format!("a received record: {e}"))),
};
if let Err(e) = validate(&r) {
return refuse(out, ShelfError::Invalid(format!("a received record: {e}")));
}
out.digest = r.digest.clone();
out.deal_id = r.deal_id.clone();
let key: LogKey = (r.sender_key.clone(), r.log_id.clone());
let held: Option<Vec<u8>> = {
let db = self.shared().db();
if tombstoned(&db, &r.digest)? {
out.deleted = true;
return Ok(out);
}
db.query_row(
"SELECT record FROM shelf_received WHERE digest = ?",
[&r.digest],
|row| row.get(0),
)
.optional()?
};
if let Some(held) = held {
if held != record {
let digest = r.digest.clone();
return refuse(
out,
ShelfError::Invalid(format!("another record is held under digest {digest}")),
);
}
out.duplicate = true;
let p = received_prefix(&self.shared().db(), &key)?;
out.counted = p.counts(r.position, &r.digest);
out.ask_for_book = !out.counted;
return Ok(out);
}
if !profile.admit(&r).map_err(ShelfError::Profile)? {
let sender = r.sender_key.clone();
return refuse(
out,
ShelfError::Refused(format!("{sender} may not push this record")),
);
}
let now = self.receive_now();
let mut attempt = 1;
let done = loop {
let mut plan = OfferPlan::default();
let mut mark = OwnMark::default();
if r.offer {
mark = read_own_mark(&self.shared().db())?;
match self.plan_offer_room(profile, &bounds, &r.sender_key, record.len() as u64)? {
Ok(p) => plan = p,
Err(refusal) => return refuse(out, refusal),
}
}
let mut part = ChangeSet::default();
match self
.receive_record_tx(&r, record, &bounds, &plan, mark, &now, &mut out, &mut part)
{
Err(TxError::Stale) if attempt < OFFER_ATTEMPTS => attempt += 1,
Err(TxError::Stale) => {
break Err(TxError::Refusal(ShelfError::Busy(
"the holder's books kept changing while an offer was being made room for"
.into(),
)))
}
Ok(()) => {
changes.merge(part);
break Ok(());
}
Err(e) => break Err(e),
}
};
match done {
Ok(()) => Ok(out),
Err(TxError::Refusal(e)) => {
out.stored = false;
out.counted = false;
out.ask_for_book = false;
refuse(out, e)
}
Err(TxError::Store(e)) => Err(e),
Err(TxError::Stale) => unreachable!("a stale mark is retried or refused as busy"),
}
}
#[allow(clippy::too_many_arguments)]
fn receive_record_tx(
&self,
r: &ReceivedRecord,
record: &[u8],
bounds: &ReceiveBounds,
plan: &OfferPlan,
mark: OwnMark,
now: &str,
out: &mut RecordOutcome,
changes: &mut ChangeSet,
) -> Result<(), TxError> {
let key: LogKey = (r.sender_key.clone(), r.log_id.clone());
self.receive_tx(|tx| {
if r.offer && read_own_mark(tx)? != mark {
return Err(TxError::Stale);
}
let count: i64 = tx.query_row(
"SELECT COUNT(*) FROM shelf_received WHERE sender_key = ? AND log_id = ?",
params![r.sender_key, r.log_id],
|row| row.get(0),
)?;
if count as u64 >= bounds.max_records_per_log {
return Err(TxError::Refusal(ShelfError::OverLimit(format!(
"{count} records held from {} in {}, the store's bound",
r.sender_key, r.log_id
))));
}
let mut before: BTreeMap<LogKey, LogState> = BTreeMap::new();
before.insert(key.clone(), read_log_state(tx, &key)?);
for digest in &plan.answered {
tx.execute(
"UPDATE shelf_received SET offer = 0 WHERE digest = ?",
[digest],
)?;
}
for row in &plan.drop {
let k = (row.sender.clone(), row.log.clone());
if let Entry::Vacant(slot) = before.entry(k) {
let st = read_log_state(tx, slot.key())?;
slot.insert(st);
}
tx.execute("DELETE FROM shelf_received WHERE digest = ?", [&row.digest])?;
}
vacate(tx, &plan.drop, true, bounds)?;
let forks = fork_evidence(tx, r)?;
tx.execute(
"INSERT INTO shelf_received (digest, sender_key, log_id, position, prev, deal_id, offer, record, received_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
params![r.digest, r.sender_key, r.log_id, sql_int(r.position)?, r.prev, r.deal_id, r.offer, record, now],
)?;
let mut offers_only = false;
if !forks.is_empty() && r.offer {
let others: i64 = tx.query_row(
"SELECT (SELECT COUNT(*) FROM shelf_received WHERE sender_key = ? AND log_id = ? AND offer = 0) + (SELECT COUNT(*) FROM shelf_received_vacated WHERE sender_key = ? AND log_id = ? AND offer = 0)",
params![r.sender_key, r.log_id, r.sender_key, r.log_id],
|row| row.get(0),
)?;
offers_only = others == 0;
}
for f in &forks {
let fresh = note_conflict(
tx,
&key,
EVIDENCE_FORK,
f.position,
&f.held,
record,
now,
offers_only,
bounds,
)?;
if fresh {
changes.forked.insert(r.deal_id.clone());
if !f.deal.is_empty() {
changes.forked.insert(f.deal.clone());
}
}
}
for (k, was) in &before {
let prefix = received_prefix(tx, k)?;
check_held_proofs(tx, k, &prefix, now, changes)?;
let after = read_log_state(tx, k)?;
report(was, &after, changes);
if *k == key {
out.counted = after.prefix.counts(r.position, &r.digest);
}
}
out.stored = true;
out.ask_for_book = !out.counted;
before_commit("record")
})
}
fn plan_offer_room(
&self,
profile: &dyn ReceiveProfile,
b: &ReceiveBounds,
sender: &str,
size: u64,
) -> Result<Result<OfferPlan, ShelfError>, ShelfError> {
if size > b.max_offer_bytes_per_sender || size > b.max_offer_bytes {
return Ok(Err(ShelfError::OverLimit(format!(
"an offer of {size} bytes, over the store's offer bound"
))));
}
struct Held {
row: ReceivedRow,
size: u64,
gone: bool,
}
let mut offers: Vec<Held> = {
let db = self.shared().db();
let mut stmt = db.prepare(&format!(
"SELECT {RECEIVED_COLUMNS}, length(record) FROM shelf_received WHERE offer = 1 ORDER BY received_at, rowid"
))?;
let rows = stmt.query_map([], |r| {
Ok(Held {
row: ReceivedRow::from_row(r)?,
size: unsigned(r, 7)?,
gone: false,
})
})?;
rows.collect::<Result<_, _>>()?
};
let mut plan = OfferPlan::default();
let scopes: [(bool, u64, u64); 2] = [
(true, b.max_offers_per_sender, b.max_offer_bytes_per_sender),
(false, b.max_offers, b.max_offer_bytes),
];
for (per_sender, max_count, max_bytes) in scopes {
loop {
let (mut count, mut bytes) = (0u64, 0u64);
let mut oldest: Option<usize> = None;
for (i, h) in offers.iter().enumerate() {
if !h.gone && (!per_sender || h.row.sender == sender) {
count += 1;
bytes += h.size;
oldest.get_or_insert(i);
}
}
if count < max_count && bytes + size <= max_bytes {
break;
}
let Some(i) = oldest else {
return Err(ShelfError::Corrupt(
"the offer bounds count offers the store does not list".into(),
));
};
offers[i].gone = true;
let row = offers[i].row.clone();
if profile
.answered(&row.record())
.map_err(ShelfError::Profile)?
{
plan.answered.push(row.digest);
} else {
plan.drop.push(row);
}
}
}
Ok(Ok(plan))
}
pub fn counted_prefix(
&self,
sender_key: &str,
log_id: &str,
) -> Result<CountedPrefix, ShelfError> {
self.live()?;
let p = received_prefix(
&self.shared().db(),
&(sender_key.to_string(), log_id.to_string()),
)?;
Ok(CountedPrefix {
size: p.counted.len() as u64,
last: p
.counted
.last()
.map(|c| c.digest.clone())
.unwrap_or_default(),
fork_at: p.fork_at,
})
}
pub fn received_record_of(&self, digest: &str) -> Result<HeldRecord, ShelfError> {
self.live()?;
let db = self.shared().db();
let row = db
.query_row(
"SELECT digest, sender_key, log_id, position, prev, deal_id, offer, record, received_at FROM shelf_received WHERE digest = ?",
[digest],
|r| {
Ok((
ReceivedRow::from_row(r)?,
r.get::<_, Vec<u8>>(7)?,
r.get::<_, String>(8)?,
))
},
)
.optional()?;
let Some((row, record, at)) = row else {
let deleted: Option<String> = db
.query_row(
"SELECT deleted_at FROM shelf_received_tombstones WHERE digest = ?",
[digest],
|r| r.get(0),
)
.optional()?;
return Err(match deleted {
None => ShelfError::NotFound(format!("received record {digest}")),
Some(at) => ShelfError::Deleted(format!("received record {digest}, at {at}")),
});
};
Ok(HeldRecord {
received: row.record(),
record,
received_at: parse_received_time(&at, &format!("received record {digest}"))?,
})
}
pub fn evidence(&self) -> Result<Vec<Evidence>, ShelfError> {
self.live()?;
let rows: Vec<(String, Vec<u8>, Vec<u8>, String)> = {
let db = self.shared().db();
let mut stmt =
db.prepare("SELECT kind, a, b, noted_at FROM shelf_evidence ORDER BY id")?;
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
rows.collect::<Result<_, _>>()?
};
rows.into_iter()
.map(|(kind, a, b, at)| {
Ok(Evidence {
kind,
a,
b,
noted_at: parse_received_time(&at, "evidence")?,
})
})
.collect()
}
pub fn forget(
&self,
deal_id: &str,
now: Time,
settled: impl FnOnce(&str) -> Result<bool, String>,
) -> Result<Vec<String>, ShelfError> {
if deal_id.is_empty() {
return Err(ShelfError::Invalid(
"forget needs a deal id and its settled check".into(),
));
}
let retention = self.shared().receive.retention;
if retention.is_zero() {
return Err(ShelfError::Invalid(
"the holder keeps received records for the life of the shelf".into(),
));
}
let _serial = self
.shared()
.receive_mu
.lock()
.unwrap_or_else(|p| p.into_inner());
self.live()?;
if !settled(deal_id).map_err(ShelfError::Profile)? {
return Err(ShelfError::Invalid(format!(
"deal {deal_id} is not settled; nothing of it is deleted"
)));
}
let cutoff = received_time(now.add_nanos(-(retention.as_nanos() as i128)));
let deleted_at = self.receive_now();
let done = self.receive_tx(|tx| {
let all: Vec<(ReceivedRow, String)> = {
let mut stmt = tx.prepare(&format!(
"SELECT {RECEIVED_COLUMNS}, received_at FROM shelf_received WHERE deal_id = ? AND received_at <= ? ORDER BY digest"
))?;
let rows = stmt.query_map(params![deal_id, cutoff], |r| {
Ok((ReceivedRow::from_row(r)?, r.get(7)?))
})?;
rows.collect::<Result<_, _>>()?
};
let mut deleted = Vec::with_capacity(all.len());
for (r, at) in &all {
tx.execute(
"INSERT INTO shelf_received_tombstones (digest, sender_key, received_at, deleted_at) VALUES (?, ?, ?, ?)",
params![r.digest, r.sender, at, deleted_at],
)?;
tx.execute("DELETE FROM shelf_received WHERE digest = ?", [&r.digest])?;
deleted.push(r.digest.clone());
}
let gone: Vec<ReceivedRow> = all.into_iter().map(|(r, _)| r).collect();
vacate(tx, &gone, false, &ReceiveBounds::default())?;
Ok(deleted)
});
match done {
Ok(deleted) => Ok(deleted),
Err(TxError::Store(e) | TxError::Refusal(e)) => Err(e),
Err(TxError::Stale) => unreachable!("forget plans no offer"),
}
}
}
fn validate(r: &ReceivedRecord) -> Result<(), &'static str> {
if !is_digest(&r.digest) {
return Err("its digest is not 64 lowercase hex characters");
}
if r.sender_key.is_empty() || r.log_id.is_empty() || r.deal_id.is_empty() {
return Err("it names no sender, log or deal");
}
if r.position == 0 {
return Err("its position is 0; the first is 1");
}
if i64::try_from(r.position).is_err() {
return Err("its position is past 2^63 - 1");
}
if r.position == 1 && !r.prev.is_empty() {
return Err("the first record names a predecessor");
}
if r.position > 1 && !is_digest(&r.prev) {
return Err("it names no predecessor digest");
}
Ok(())
}