use std::collections::HashMap;
use std::fmt;
use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use crate::journal::frame::SaturatingFrom;
use std::sync::{Arc, Mutex};
use lgwks_std::hash::{Digest, Hasher};
use lgwks_std::wire::{WireError, from_bytes, to_bytes};
use crate::effect::RunId;
use crate::journal::frame::{self, Cursor, HEAD_BYTES};
use crate::journal::owner::{self, Stage, StorageOwner, SubmitError};
use super::store::{StoreError, StoreLimitKind};
const LEDGER_MAGIC: &[u8; 17] = b"lgwks-runledger\x00\x01";
pub const MAX_LEDGER_RECORD_BYTES: usize = 16 * 1024;
pub const MAX_LEDGER_RECORDS_PER_RUN: u64 = 65_536;
fn genesis_head() -> Digest {
let mut hasher = Hasher::new();
hasher.write_framed(b"lgwks-runledger/genesis");
hasher.finalize()
}
#[derive(
Debug, Clone, lgwks_std::wire::Archive, lgwks_std::wire::Serialize, lgwks_std::wire::Deserialize,
)]
#[rkyv(
attr(non_exhaustive),
crate = lgwks_std::wire::rkyv,
compare(PartialEq),
derive(Debug)
)]
struct Entry {
#[rkyv(attr(doc = "The run this entry belongs to."))]
run: RunId,
#[rkyv(attr(doc = "The tenant that owns the run."))]
tenant: String,
#[rkyv(attr(doc = "The root attempts this run has been charged, in total."))]
attempts: u64,
#[rkyv(attr(doc = "The root spend this run has been charged, in total."))]
spend: u64,
#[rkyv(attr(doc = "The repair epoch this entry leaves the run at."))]
epoch: u64,
#[rkyv(attr(doc = "The repair ticket identity this entry applied, if any."))]
applied: Option<Vec<u8>>,
}
impl Entry {
fn head_from(&self, previous: &Digest) -> Digest {
let mut hasher = Hasher::new();
hasher.write_framed(previous.as_bytes());
hasher.write_framed(self.run.id().to_hex().as_bytes());
hasher.write_framed(self.tenant.as_bytes());
hasher.write_framed(&self.attempts.to_le_bytes());
hasher.write_framed(&self.spend.to_le_bytes());
hasher.write_framed(&self.epoch.to_le_bytes());
hasher.write_framed(match self.applied {
Some(ref ticket) => ticket.as_slice(),
None => &[],
});
hasher.finalize()
}
}
#[derive(Debug)]
struct Index {
runs: HashMap<RunId, Control>,
committed: u64,
tail: Digest,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Control {
tenant: String,
attempts: u64,
spend: u64,
epoch: u64,
applied: Vec<Vec<u8>>,
}
impl Control {
fn fresh(tenant: &str) -> Self {
Self {
tenant: tenant.to_owned(),
attempts: 0,
spend: 0,
epoch: 0,
applied: Vec::new(),
}
}
#[must_use]
pub fn tenant(&self) -> &str {
&self.tenant
}
#[must_use]
pub const fn attempts(&self) -> u64 {
self.attempts
}
#[must_use]
pub const fn spend(&self) -> u64 {
self.spend
}
#[must_use]
pub const fn epoch(&self) -> u64 {
self.epoch
}
#[must_use]
pub fn applied(&self) -> usize {
self.applied.len()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TicketStamp {
pub(crate) identity: Vec<u8>,
pub(crate) epoch: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum LeaseRefusal {
ForeignTenant {
owner: String,
asked: String,
},
StaleEpoch {
current: u64,
offered: u64,
},
AlreadyApplied,
BudgetSpent {
attempts: u64,
max_attempts: u64,
},
}
impl std::error::Error for LeaseRefusal {}
impl fmt::Display for LeaseRefusal {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
Self::ForeignTenant {
ref owner,
ref asked,
} => write!(
formatter,
"run belongs to tenant {owner:?}, not {asked:?}; refusing to read its repair ledger"
),
Self::StaleEpoch { current, offered } => write!(
formatter,
"repair epoch {offered} is not this run's current epoch {current}; \
refusing a stale repair ticket"
),
Self::AlreadyApplied => formatter.write_str(
"this repair ticket was already applied to this run; refusing to apply it twice",
),
Self::BudgetSpent {
attempts,
max_attempts,
} => write!(
formatter,
"root budget spent: {attempts} attempts against a ceiling of {max_attempts}; \
an authorized repair does not refill it"
),
}
}
}
#[derive(Clone)]
pub struct RunLedger {
inner: Arc<Inner>,
}
struct Inner {
owner: StorageOwner<Arc<Mutex<Index>>, Control>,
path: PathBuf,
index: Arc<Mutex<Index>>,
}
impl RunLedger {
pub fn open(path: impl Into<PathBuf>) -> Result<Self, StoreError> {
let path = path.into();
let existed = path.exists();
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)
.map_err(StoreError::storage)?;
if !existed || file.metadata().map_err(StoreError::storage)?.len() == 0 {
file.write_all(LEDGER_MAGIC)
.and_then(|()| file.sync_all())
.map_err(StoreError::storage)?;
}
let index = replay(&mut file)?;
if file.metadata().map_err(StoreError::storage)?.len() != index.committed {
file.set_len(index.committed).map_err(StoreError::storage)?;
file.sync_all().map_err(StoreError::storage)?;
}
let index = Arc::new(Mutex::new(index));
let owner =
StorageOwner::spawn(file, Arc::clone(&index), false).map_err(StoreError::storage)?;
Ok(Self {
inner: Arc::new(Inner { owner, path, index }),
})
}
pub fn open_in(dir: &Path, tenant: &str) -> Result<Self, StoreError> {
std::fs::create_dir_all(dir).map_err(StoreError::storage)?;
Self::open(dir.join(format!("{tenant}.runledger")))
}
#[must_use]
pub fn path(&self) -> &Path {
&self.inner.path
}
#[must_use]
pub fn control(&self, run: RunId) -> Option<Control> {
owner::lock(&self.inner.index).runs.get(&run).cloned()
}
pub(crate) async fn charge(
&self,
tenant: &str,
run: RunId,
ticket: Option<&TicketStamp>,
spend: u64,
max_attempts: u64,
max_spend: u64,
) -> Result<Control, CommitError> {
let charge = Charge {
tenant: tenant.to_owned(),
run,
ticket: ticket.cloned(),
cost: spend,
};
let ceilings = Ceilings {
attempts: max_attempts,
spend: max_spend,
};
self.inner
.owner
.enqueue_awaiting(move |file, shared| charge_on_owner(file, shared, &charge, ceilings))
.await
.map_err(classify)
}
}
fn classify(cause: SubmitError) -> CommitError {
let is_ours = matches!(
cause,
SubmitError::Device(ref error)
if error
.get_ref()
.and_then(<dyn std::error::Error + Send + Sync>::downcast_ref::<LeaseRefusal>)
.is_some()
);
if !is_ours {
return CommitError::Store(StoreError::from(cause));
}
match cause {
SubmitError::Device(error) => match error.into_inner() {
Some(inner) => match inner.downcast::<LeaseRefusal>() {
Ok(refusal) => CommitError::Refused(*refusal),
Err(other) => CommitError::Store(StoreError::storage(std::io::Error::other(other))),
},
None => CommitError::Store(StoreError::storage(std::io::Error::other(
"the ledger owner refused a charge without naming why",
))),
},
other => CommitError::Store(StoreError::from(other)),
}
}
fn charge_on_owner(
file: &mut File,
shared: &Arc<Mutex<Index>>,
charge: &Charge,
ceilings: Ceilings,
) -> std::io::Result<Stage<Control, Arc<Mutex<Index>>>> {
let mut index = owner::lock(shared);
let (entry, next) = decide_under(&index, charge, ceilings).map_err(std::io::Error::other)?;
write_entry(file, &mut index, &entry)
.map_err(|error| std::io::Error::other(error.to_string()))?;
Ok(Stage::Committed(next))
}
#[derive(Clone, Debug)]
pub(crate) struct Charge {
pub(crate) tenant: String,
pub(crate) run: RunId,
pub(crate) ticket: Option<TicketStamp>,
pub(crate) cost: u64,
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct Ceilings {
pub(crate) attempts: u64,
pub(crate) spend: u64,
}
fn decide_under(
index: &Index,
charge: &Charge,
ceilings: Ceilings,
) -> Result<(Entry, Control), LeaseRefusal> {
let tenant = charge.tenant.as_str();
let run = &charge.run;
let ticket = charge.ticket.as_ref();
let cost = charge.cost;
let max_attempts = ceilings.attempts;
let max_spend = ceilings.spend;
let mut next = Control::fresh(tenant);
if let Some(control) = index.runs.get(run) {
next = control.clone();
}
if next.tenant != tenant {
let refusal = Err(LeaseRefusal::ForeignTenant {
owner: next.tenant,
asked: tenant.to_owned(),
});
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "decide_under: returning an error to the caller");
return refusal;
}
let next_attempts = next.attempts.saturating_add(1);
let next_spend = next.spend.saturating_add(cost);
if next_attempts > max_attempts || next_spend > max_spend {
let refusal = Err(LeaseRefusal::BudgetSpent {
attempts: next_attempts,
max_attempts,
});
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "decide_under: returning an error to the caller");
return refusal;
}
let mut applied = None;
if let Some(stamp) = ticket {
if stamp.epoch != next.epoch {
let refusal = Err(LeaseRefusal::StaleEpoch {
current: next.epoch,
offered: stamp.epoch,
});
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "decide_under: returning an error to the caller");
return refusal;
}
if next.applied.contains(&stamp.identity) {
let refusal = Err(LeaseRefusal::AlreadyApplied);
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "decide_under: returning an error to the caller");
return refusal;
}
next.epoch = next.epoch.saturating_add(1);
next.applied.push(stamp.identity.clone());
applied = Some(stamp.identity.clone());
}
next.attempts = next_attempts;
next.spend = next_spend;
let entry = Entry {
run: *run,
tenant: tenant.to_owned(),
attempts: next.attempts,
spend: next.spend,
epoch: next.epoch,
applied,
};
Ok((entry, next))
}
fn write_entry(file: &mut File, index: &mut Index, entry: &Entry) -> Result<(), StoreError> {
let previous = index.tail;
let (framed, head) = frame(entry, &previous)?;
let staged = u64::saturating_from(framed.len());
let next = index
.committed
.checked_add(staged)
.ok_or(StoreError::Limit {
kind: StoreLimitKind::StoreBytes,
requested: u64::MAX,
limit: super::MAX_STORE_BYTES,
})?;
let on_disk = file.metadata().map_err(StoreError::storage)?.len();
if on_disk != index.committed {
let refusal = Err(StoreError::Corrupt { at: 0 });
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "write_entry: returning an error to the caller");
return refusal;
}
file.write_all(&framed).map_err(StoreError::storage)?;
file.sync_all().map_err(StoreError::storage)?;
index.committed = next;
index.tail = head;
fold(index, entry);
Ok(())
}
fn fold(index: &mut Index, entry: &Entry) {
let run = index.runs.entry(entry.run).or_insert_with(|| Control {
tenant: entry.tenant.clone(),
attempts: 0,
spend: 0,
epoch: 0,
applied: Vec::new(),
});
run.tenant.clone_from(&entry.tenant);
run.attempts = entry.attempts;
run.spend = entry.spend;
run.epoch = entry.epoch;
if let Some(ref ticket) = entry.applied
&& !run.applied.contains(ticket)
{
run.applied.push(ticket.clone());
}
}
fn frame(entry: &Entry, previous: &Digest) -> Result<(Vec<u8>, Digest), StoreError> {
frame::frame_record(
entry,
previous,
MAX_LEDGER_RECORD_BYTES,
|entry| {
to_bytes::<WireError>(entry)
.map(|bytes| bytes.as_ref().to_vec())
.map_err(|cause| StoreError::Encoding { cause })
},
|entry, previous, _| entry.head_from(previous),
|len| StoreError::Limit {
kind: StoreLimitKind::RecordBytes,
requested: u64::saturating_from(len),
limit: u64::saturating_from(MAX_LEDGER_RECORD_BYTES),
},
)
}
struct Framed {
entry: Entry,
head: [u8; HEAD_BYTES],
declared: usize,
}
fn next_frame(file: &mut File, cursor: &Cursor<'_>) -> Result<Option<Framed>, StoreError> {
let at = cursor.at;
let corrupt = || StoreError::Corrupt { at };
let ceiling = MAX_LEDGER_RECORD_BYTES;
let entry_head = |previous: &Digest, payload: &[u8]| {
let aligned = frame::decodable(payload);
from_bytes::<Entry, WireError>(aligned.as_slice())
.ok()
.map(|entry| entry.head_from(previous))
};
let Some(raw) = frame::read_raw(
file,
cursor,
ceiling,
StoreError::storage,
corrupt,
entry_head,
)?
else {
return Ok(None);
};
let entry = from_bytes::<Entry, WireError>(&raw.payload).map_err(|error| {
lgwks_std::trace::debug!(?error, at, "next_frame: the payload did not decode");
StoreError::Corrupt { at }
})?;
Ok(Some(Framed {
entry,
declared: raw.payload.len(),
head: raw.head,
}))
}
fn replay(file: &mut File) -> Result<Index, StoreError> {
let total = file.metadata().map_err(StoreError::storage)?.len();
file.seek(SeekFrom::Start(0)).map_err(StoreError::storage)?;
let mut header = [0u8; LEDGER_MAGIC.len()];
if !read_full(file, &mut header)? || header != *LEDGER_MAGIC {
let refusal = Err(StoreError::NotAStore);
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "replay: returning an error to the caller");
return refusal;
}
let mut index = Index {
runs: HashMap::new(),
committed: u64::saturating_from(LEDGER_MAGIC.len()),
tail: genesis_head(),
};
let mut previous = genesis_head();
let mut at = 0u64;
while let Some(Framed {
entry,
head,
declared,
}) = next_frame(file, &Cursor::new(at, index.committed, &previous))?
{
if entry.head_from(&previous) != Digest::from_bytes(head) {
let refusal = Err(StoreError::Corrupt { at });
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "replay: returning an error to the caller");
return refusal;
}
let frame_len = frame::framed_len(declared);
if index
.committed
.checked_add(frame_len)
.is_none_or(|end| end > total)
{
break;
}
index.committed = index.committed.saturating_add(frame_len);
index.tail = entry.head_from(&previous);
previous = index.tail;
at = at.saturating_add(1);
if at > MAX_LEDGER_RECORDS_PER_RUN {
let refusal = Err(StoreError::Limit {
kind: StoreLimitKind::Records,
requested: at,
limit: MAX_LEDGER_RECORDS_PER_RUN,
});
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "replay: returning an error to the caller");
return refusal;
}
fold(&mut index, &entry);
}
Ok(index)
}
fn read_full(reader: &mut impl Read, buf: &mut [u8]) -> Result<bool, StoreError> {
let mut filled = 0usize;
while filled < buf.len() {
match reader.read(&mut buf[filled..]) {
Ok(0) => return Ok(false),
Ok(read) => filled = filled.saturating_add(read),
Err(ref error) if error.kind() == std::io::ErrorKind::Interrupted => {}
Err(cause) => {
let refusal = Err(StoreError::storage(cause));
lgwks_std::trace::debug!(error = ?refusal.as_ref().err(), "read_full: returning an error to the caller");
return refusal;
}
}
}
Ok(true)
}
#[derive(Debug)]
pub enum CommitError {
Store(StoreError),
Refused(LeaseRefusal),
}
impl fmt::Display for CommitError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
Self::Store(ref cause) => write!(formatter, "{cause}"),
Self::Refused(ref cause) => write!(formatter, "{cause}"),
}
}
}
impl std::error::Error for CommitError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match *self {
Self::Store(ref cause) => Some(cause),
Self::Refused(_) => None,
}
}
}
impl fmt::Debug for RunLedger {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RunLedger")
.field("path", &self.inner.path)
.field("runs", &owner::lock(&self.inner.index).runs.len())
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::journal::frame::probe::{Scratch, declared_at, frame_starts, with_prefix};
type TestResult = Result<(), Box<dyn std::error::Error>>;
fn written(path: &Path, count: u64) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
let run = RunId::from_hex(&format!("b{}", "0".repeat(31)))?;
let mut bytes = LEDGER_MAGIC.to_vec();
let mut previous = genesis_head();
for charge in 1..=count {
let entry = Entry {
run,
tenant: "acme".to_owned(),
attempts: charge,
spend: charge.saturating_mul(3),
epoch: charge,
applied: Some(vec![u8::try_from(charge)?; 24]),
};
let (framed, head) = frame(&entry, &previous)?;
bytes.extend_from_slice(&framed);
previous = head;
}
std::fs::write(path, &bytes)?;
Ok(bytes)
}
#[test]
fn a_lengthened_acknowledged_final_entry_is_refused_not_trimmed() -> TestResult {
let scratch = Scratch::new("ledger-lengthened")?;
let bytes = written(scratch.path(), 3)?;
let last = frame_starts(&bytes, LEDGER_MAGIC.len())?[2];
let declared = declared_at(&bytes, last);
for extra in (1u32..=40).chain([100, 1024]) {
let lied = with_prefix(&bytes, last, declared + extra);
std::fs::write(scratch.path(), &lied)?;
assert!(
matches!(
RunLedger::open(scratch.path()),
Err(StoreError::Corrupt { at: 2 })
),
"L+{extra}: an acknowledged entry must be refused as corrupt at 2"
);
assert_eq!(
std::fs::read(scratch.path())?,
lied,
"L+{extra}: bytes move"
);
}
Ok(())
}
#[test]
fn a_damaged_lengthened_entry_with_an_acknowledged_one_behind_it_is_refused() -> TestResult {
let scratch = Scratch::new("ledger-damaged")?;
let bytes = written(scratch.path(), 3)?;
let middle = frame_starts(&bytes, LEDGER_MAGIC.len())?[1];
let mut lied = with_prefix(&bytes, middle, u32::try_from(bytes.len() - middle)? + 9);
if let Some(byte) = lied.get_mut(middle + 10) {
*byte ^= 0x55;
}
std::fs::write(scratch.path(), &lied)?;
assert!(matches!(
RunLedger::open(scratch.path()),
Err(StoreError::Corrupt { at: 1 })
));
assert_eq!(std::fs::read(scratch.path())?, lied, "refused bytes move");
Ok(())
}
#[test]
fn an_append_cut_inside_the_final_entry_is_repaired() -> TestResult {
let scratch = Scratch::new("ledger-cut")?;
let bytes = written(scratch.path(), 3)?;
let last = frame_starts(&bytes, LEDGER_MAGIC.len())?[2];
let whole = bytes.len() - last;
for cut in [1, 3, 4, 5, whole >> 1, whole - 33, whole - 32, whole - 1] {
std::fs::write(
scratch.path(),
bytes.get(..last + cut).ok_or("past the end")?,
)?;
drop(RunLedger::open(scratch.path())?);
assert_eq!(
std::fs::metadata(scratch.path())?.len(),
u64::try_from(last)?,
"cut {cut}: repaired to the acknowledged prefix"
);
}
Ok(())
}
}