use std::{
collections::{BTreeMap, HashMap},
fmt,
path::Path,
};
use jiff::Timestamp;
use miette::Result;
use uuid::Uuid;
use super::{
read::{Range, Reader, Renamed, Skipped, Unreadable},
record::{GapRecord, Record, RecordKind},
writer::{BACKLOG_BYTES, BACKLOG_RECORDS},
};
const REORDER_LIMIT: usize = BACKLOG_RECORDS * 10;
const REORDER_BYTES: usize = BACKLOG_BYTES;
const RUN_BYTES: usize = std::mem::size_of::<(u64, u64)>() * 4;
const MOST_GAPS_REPORTED: usize = 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Break {
pub seq: u64,
pub ts: Timestamp,
pub reason: BreakReason,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BreakReason {
Unchained { expected: String, found: String },
Missing { expected: u64 },
}
impl fmt::Display for BreakReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Unchained { expected, found } => {
let found = if found.is_empty() { "nothing" } else { found };
write!(f, "chains onto {found}, expected {expected}")
}
Self::Missing { expected } => {
write!(f, "sequence number {expected} is unaccounted for")
}
}
}
}
#[derive(Debug, Clone)]
pub struct ChainReport {
pub instance: Uuid,
pub records: u64,
pub broken_at: Option<Break>,
pub gaps: Vec<(u64, GapRecord)>,
pub gaps_beyond: u64,
pub truncated_start: bool,
pub head: String,
}
impl ChainReport {
pub fn holds(&self) -> bool {
self.broken_at.is_none()
}
pub fn lost(&self) -> u64 {
self.gaps.iter().map(|(_, gap)| gap.lost).sum()
}
}
#[derive(Debug, Clone, Default)]
pub struct VerifyReport {
pub sessions: Vec<ChainReport>,
pub skipped: Vec<Skipped>,
pub unattributed: u64,
pub renamed: Vec<Renamed>,
pub unreadable: Vec<Unreadable>,
}
impl VerifyReport {
pub fn holds(&self) -> bool {
self.unattributed == 0
&& self.renamed.is_empty()
&& self.unreadable.is_empty()
&& self.sessions.iter().all(ChainReport::holds)
}
pub fn heads(&self) -> impl Iterator<Item = (Uuid, &str)> {
self.sessions
.iter()
.map(|session| (session.instance, session.head.as_str()))
}
}
#[derive(Debug)]
struct Held {
record: Record,
hash: String,
weight: usize,
}
#[derive(Debug)]
struct Chain {
records: u64,
head: String,
broken_at: Option<Break>,
gaps: Vec<(u64, GapRecord)>,
gaps_beyond: u64,
truncated_start: bool,
covered_to: u64,
ahead: BTreeMap<u64, u64>,
first_ahead: Option<(u64, Timestamp)>,
pending: HashMap<String, Held>,
pending_bytes: usize,
}
impl Chain {
fn start(record: &Record, hash: &str) -> Self {
let truncated_start = !(record.prev.is_empty() && record.seq == 0);
let mut chain = Self {
records: 0,
head: String::new(),
broken_at: None,
gaps: Vec::new(),
gaps_beyond: 0,
truncated_start,
covered_to: record.seq,
ahead: BTreeMap::new(),
first_ahead: None,
pending: HashMap::new(),
pending_bytes: 0,
};
chain.consume(record, hash);
chain
}
fn consume(&mut self, record: &Record, hash: &str) {
self.records += 1;
let through = match &record.kind {
RecordKind::Gap(gap) => {
if self.gaps.len() < MOST_GAPS_REPORTED {
self.gaps.push((record.seq, gap.clone()));
} else {
self.gaps_beyond += 1;
}
gap.through.max(record.seq)
}
_ => record.seq,
};
self.account(record.seq, through, record.ts);
self.head = hash.to_owned();
}
fn account(&mut self, from: u64, through: u64, ts: Timestamp) {
if through < self.covered_to {
return;
}
if from > self.covered_to {
if self.first_ahead.is_none() {
self.first_ahead = Some((from, ts));
}
if self.ahead.len() < REORDER_LIMIT {
let end = self.ahead.entry(from).or_insert(through);
*end = (*end).max(through);
self.pending_bytes += RUN_BYTES;
}
} else {
self.covered_to = through.saturating_add(1);
}
while let Some((&start, &end)) = self.ahead.range(..=self.covered_to).next_back() {
self.ahead.remove(&start);
self.pending_bytes = self.pending_bytes.saturating_sub(RUN_BYTES);
self.covered_to = self.covered_to.max(end.saturating_add(1));
}
if self.ahead.is_empty() {
self.first_ahead = None;
}
}
fn offer(&mut self, record: Record, hash: String, weight: usize) -> isize {
if record.prev == self.head {
self.consume(&record, &hash);
return -(self.drain() as isize);
}
if self.pending.len() >= REORDER_LIMIT {
self.note_break(&record);
return 0;
}
if self.pending.contains_key(&record.prev) {
self.note_break(&record);
return 0;
}
self.pending_bytes += weight;
self.pending.insert(
record.prev.clone(),
Held {
record,
hash,
weight,
},
);
weight as isize
}
fn relinquish(&mut self) -> usize {
let Some(prev) = self
.pending
.iter()
.max_by_key(|(_, held)| held.weight)
.map(|(prev, _)| prev.clone())
else {
return 0;
};
let Some(held) = self.pending.remove(&prev) else {
return 0;
};
self.pending_bytes = self.pending_bytes.saturating_sub(held.weight);
self.note_break(&held.record);
held.weight
}
fn drain(&mut self) -> usize {
let mut freed = 0;
while let Some(held) = self.pending.remove(&self.head) {
freed += held.weight;
self.pending_bytes = self.pending_bytes.saturating_sub(held.weight);
self.consume(&held.record, &held.hash);
}
freed
}
fn note_break(&mut self, record: &Record) {
if self.broken_at.is_none() {
self.broken_at = Some(Break {
seq: record.seq,
ts: record.ts,
reason: BreakReason::Unchained {
expected: self.head.clone(),
found: record.prev.clone(),
},
});
}
self.records += 1;
}
fn finish(mut self, instance: Uuid) -> ChainReport {
if let Some((seq, ts)) = self.first_ahead
&& self.broken_at.is_none()
{
self.broken_at = Some(Break {
seq,
ts,
reason: BreakReason::Missing {
expected: self.covered_to,
},
});
}
if !self.pending.is_empty() {
let orphan = self
.pending
.values()
.min_by_key(|held| (held.record.seq, held.record.ts))
.map(|held| held.record.clone());
if let Some(orphan) = orphan {
self.note_break(&orphan);
}
self.records += self.pending.len() as u64 - 1;
}
ChainReport {
instance,
records: self.records,
broken_at: self.broken_at,
gaps: self.gaps,
gaps_beyond: self.gaps_beyond,
truncated_start: self.truncated_start,
head: self.head,
}
}
}
pub fn verify(dir: &Path) -> Result<VerifyReport> {
verify_range(dir, Range::default())
}
pub fn verify_range(dir: &Path, range: Range) -> Result<VerifyReport> {
let mut reader = Reader::open_range(dir, range)?;
let mut chains: HashMap<Uuid, Chain> = HashMap::new();
let mut order: Vec<Uuid> = Vec::new();
let mut unattributed = 0;
let mut held_bytes = 0usize;
for stored in reader.by_ref() {
let Some(instance) = stored.instance else {
unattributed += 1;
continue;
};
let weight = stored.json.len();
match chains.get_mut(&instance) {
Some(chain) => {
let change = chain.offer(stored.record, stored.hash, weight);
held_bytes = held_bytes.saturating_add_signed(change);
}
None => {
order.push(instance);
chains.insert(instance, Chain::start(&stored.record, &stored.hash));
}
}
while held_bytes > REORDER_BYTES {
let Some(worst) = chains
.iter_mut()
.max_by_key(|(_, chain)| chain.pending_bytes)
.map(|(instance, _)| *instance)
else {
break;
};
let Some(chain) = chains.get_mut(&worst) else {
break;
};
let freed = chain.relinquish();
if freed == 0 {
break;
}
held_bytes = held_bytes.saturating_sub(freed);
}
}
let skipped = reader.skipped().to_vec();
let renamed = reader.renamed().to_vec();
let unreadable = reader.unreadable().to_vec();
let sessions = order
.into_iter()
.filter_map(|instance| chains.remove(&instance).map(|chain| chain.finish(instance)))
.collect();
Ok(VerifyReport {
sessions,
skipped,
unattributed,
renamed,
unreadable,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::audit::{
record::{QuerySource, frame},
writer::{Context, Writer},
};
use jiff::Timestamp;
fn context() -> Context {
Context {
sys_user: "felix".into(),
db_user: "tamanu".into(),
writemode: false,
ots: None,
}
}
fn write_session(dir: &Path, queries: usize) -> Uuid {
let mut writer = Writer::new(dir);
let instance = writer.instance();
for i in 0..queries {
writer.query(&context(), format!("select {i};"), QuerySource::Typed);
}
instance
}
fn segment_of(dir: &Path, instance: Uuid) -> std::path::PathBuf {
crate::audit::paths::list(dir)
.unwrap()
.into_iter()
.find(|(_, kind)| kind.instance() == Some(instance))
.map(|(path, _)| path)
.unwrap()
}
#[test]
fn a_clean_session_verifies() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 5);
let report = verify(dir.path()).unwrap();
assert!(report.holds());
assert_eq!(report.sessions.len(), 1);
assert_eq!(report.sessions[0].instance, instance);
assert!(!report.sessions[0].truncated_start);
assert!(report.sessions[0].gaps.is_empty());
assert!(!report.sessions[0].head.is_empty());
assert_eq!(report.unattributed, 0);
}
#[test]
fn concurrent_sessions_each_verify() {
let dir = tempfile::tempdir().unwrap();
let mut one = Writer::new(dir.path());
let mut two = Writer::new(dir.path());
for i in 0..10 {
one.query(&context(), format!("one {i};"), QuerySource::Typed);
two.query(&context(), format!("two {i};"), QuerySource::Typed);
}
drop(one);
drop(two);
let report = verify(dir.path()).unwrap();
assert_eq!(report.sessions.len(), 2);
assert!(report.holds());
}
#[test]
fn altering_a_record_in_place_breaks_the_chain() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 5);
let path = segment_of(dir.path(), instance);
let text = std::fs::read_to_string(&path).unwrap();
std::fs::write(&path, text.replace("select 2;", "select 9;")).unwrap();
let report = verify(dir.path()).unwrap();
assert!(!report.holds());
let broken = report.sessions[0].broken_at.as_ref().unwrap();
assert!(matches!(broken.reason, BreakReason::Unchained { .. }));
}
#[test]
fn removing_a_record_breaks_the_chain() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 5);
let path = segment_of(dir.path(), instance);
let bytes = std::fs::read(&path).unwrap();
let text = String::from_utf8(bytes).unwrap();
let kept: String = text
.split('\u{1e}')
.enumerate()
.filter(|(i, _)| *i != 3)
.map(|(_, part)| format!("\u{1e}{part}"))
.collect::<String>()
.replacen('\u{1e}', "", 1);
std::fs::write(&path, kept).unwrap();
let report = verify(dir.path()).unwrap();
assert!(!report.holds());
}
fn session_over_three_days(dir: &Path) -> (Uuid, Vec<std::path::PathBuf>) {
let mut writer = Writer::new(dir);
let instance = writer.instance();
writer.query(&context(), "day one;".into(), QuerySource::Typed);
drop(writer);
let today = crate::audit::writer::date_of(Timestamp::now());
let first = dir.join(crate::audit::paths::segment_name(today, instance));
let mut paths = vec![first.clone()];
let mut prev_records = crate::audit::writer::read_records(&first);
for day in 1..3 {
let date = today.checked_add(jiff::Span::new().days(day)).unwrap();
let previous = prev_records.last().unwrap().clone();
let record = Record {
v: crate::audit::record::FORMAT_VERSION,
seq: previous.seq + 1,
ts: previous.ts + jiff::SignedDuration::from_hours(24),
prev: crate::audit::record::hash(&previous.to_json().unwrap()),
kind: RecordKind::Query(crate::audit::record::QueryRecord {
query: format!("day {};", day + 1),
source: QuerySource::Typed,
}),
};
let path = dir.join(crate::audit::paths::segment_name(date, instance));
std::fs::write(&path, frame(&record.to_json().unwrap())).unwrap();
prev_records = vec![record];
paths.push(path);
}
(instance, paths)
}
#[test]
fn removing_a_segment_from_the_middle_breaks_the_session_it_belonged_to() {
let dir = tempfile::tempdir().unwrap();
let (instance, paths) = session_over_three_days(dir.path());
assert!(verify(dir.path()).unwrap().holds());
std::fs::remove_file(&paths[1]).unwrap();
let report = verify(dir.path()).unwrap();
let session = report
.sessions
.iter()
.find(|s| s.instance == instance)
.unwrap();
assert!(
!session.holds(),
"a whole segment cannot go missing unnoticed"
);
}
#[test]
fn losing_the_oldest_records_reads_as_retention_rather_than_tampering() {
let dir = tempfile::tempdir().unwrap();
let (instance, paths) = session_over_three_days(dir.path());
std::fs::remove_file(&paths[0]).unwrap();
let report = verify(dir.path()).unwrap();
let session = report
.sessions
.iter()
.find(|s| s.instance == instance)
.unwrap();
assert!(session.truncated_start);
assert!(session.holds());
}
#[test]
fn a_gap_is_reported_without_failing_the_chain() {
let dir = tempfile::tempdir().unwrap();
let store = dir.path().join("store");
std::fs::write(&store, b"in the way").unwrap();
let mut writer = Writer::new(&store);
for i in 0..(crate::audit::writer::BACKLOG_RECORDS + 50) {
writer.query(&context(), format!("lost {i};"), QuerySource::Typed);
}
std::fs::remove_file(&store).unwrap();
writer.query(&context(), "after;".into(), QuerySource::Typed);
drop(writer);
let report = verify(&store).unwrap();
assert_eq!(report.sessions.len(), 1);
let session = &report.sessions[0];
assert!(!session.gaps.is_empty(), "the gap is reported");
assert!(session.lost() >= 50);
assert!(
session.holds(),
"an incomplete log is not an altered one: {:?}",
session.broken_at
);
}
#[test]
fn unparsable_bytes_are_reported() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 3);
let path = segment_of(dir.path(), instance);
let mut bytes = std::fs::read(&path).unwrap();
bytes.push(0x1E);
bytes.extend_from_slice(b"half a record");
std::fs::write(&path, bytes).unwrap();
let report = verify(dir.path()).unwrap();
assert_eq!(report.skipped.len(), 1);
assert!(
report.holds(),
"a torn record does not break the records around it"
);
}
#[test]
fn a_crafted_gap_record_does_not_crash_the_verifier() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 2);
let path = segment_of(dir.path(), instance);
let text = std::fs::read_to_string(&path).unwrap();
let crafted = format!(
r#"{{"v":1,"seq":1,"ts":"2026-09-08T00:00:00Z","prev":"","kind":"gap","lost":1,"through":{},"from":"2026-09-08T00:00:00Z","to":"2026-09-08T00:00:00Z"}}"#,
u64::MAX
);
std::fs::write(&path, format!("{text}\u{1e}{crafted}\n")).unwrap();
let report = verify(dir.path()).unwrap();
assert!(!report.sessions.is_empty());
}
#[test]
fn a_record_belonging_to_no_session_fails_the_log() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 3);
let path = segment_of(dir.path(), instance);
let records = crate::audit::writer::read_records(&path);
let spliced = Record {
v: crate::audit::record::FORMAT_VERSION,
seq: 99,
ts: Timestamp::now(),
prev: "0".repeat(64),
kind: RecordKind::Query(crate::audit::record::QueryRecord {
query: "spliced;".into(),
source: QuerySource::Typed,
}),
};
let day = dir.path().join(crate::audit::paths::day_file_name(
crate::audit::writer::date_of(records[0].ts),
));
let mut bytes = Vec::new();
for record in records.iter().chain(std::iter::once(&spliced)) {
bytes.extend_from_slice(&frame(&record.to_json().unwrap()));
}
std::fs::write(&day, zstd::encode_all(&bytes[..], 3).unwrap()).unwrap();
std::fs::remove_file(&path).unwrap();
let report = verify(dir.path()).unwrap();
assert_eq!(report.unattributed, 1);
assert!(
!report.holds(),
"a record no session wrote cannot pass as one that was"
);
}
#[test]
fn renaming_a_segment_does_not_re_attribute_what_is_in_it() {
let dir = tempfile::tempdir().unwrap();
let wrote = write_session(dir.path(), 3);
let path = segment_of(dir.path(), wrote);
let someone_else = Uuid::new_v4();
let moved = dir.path().join(crate::audit::paths::segment_name(
crate::audit::writer::date_of(Timestamp::now()),
someone_else,
));
std::fs::rename(&path, &moved).unwrap();
let report = verify(dir.path()).unwrap();
assert_eq!(report.renamed.len(), 1);
assert_eq!(report.renamed[0].names, someone_else);
assert_eq!(report.renamed[0].records, wrote);
assert!(!report.holds(), "a rename cannot pass unremarked");
assert_eq!(report.sessions[0].instance, wrote);
}
#[test]
fn a_flood_of_unchainable_records_is_reported_rather_than_held() {
use crate::audit::record::{ContextRecord, FORMAT_VERSION, frame};
let dir = tempfile::tempdir().unwrap();
let instance = Uuid::new_v4();
let big = "x".repeat(64 * 1024);
let mut bytes = Vec::new();
for seq in 0..600u64 {
let record = Record {
v: FORMAT_VERSION,
seq,
ts: Timestamp::UNIX_EPOCH + jiff::SignedDuration::from_secs(seq as i64),
prev: format!("{seq:064}"),
kind: RecordKind::Context(ContextRecord {
sys_user: big.clone(),
db_user: "tamanu".into(),
writemode: false,
ots: Some(big.clone()),
tailscale: Vec::new(),
instance,
}),
};
bytes.extend_from_slice(&frame(&record.to_json().unwrap()));
}
std::fs::write(
dir.path().join(crate::audit::paths::segment_name(
"2026-09-08".parse().unwrap(),
instance,
)),
bytes,
)
.unwrap();
let report = verify(dir.path()).unwrap();
assert!(!report.holds(), "a chain that cannot hold is reported");
}
#[test]
fn a_file_that_stops_being_readable_fails_the_log() {
let dir = tempfile::tempdir().unwrap();
write_session(dir.path(), 3);
let truncated = dir.path().join(crate::audit::paths::day_file_name(
"2026-09-08".parse().unwrap(),
));
let mut bytes = zstd::encode_all(&b"\x1e{\"v\":1}\n"[..], 3).unwrap();
bytes.truncate(bytes.len() / 2);
std::fs::write(&truncated, bytes).unwrap();
let report = verify(dir.path()).unwrap();
assert_eq!(report.unreadable.len(), 1);
assert!(
!report.holds(),
"a log that could not be read through is not a log that holds"
);
}
#[test]
fn two_records_claiming_the_same_predecessor_are_both_counted() {
let dir = tempfile::tempdir().unwrap();
let instance = write_session(dir.path(), 3);
let path = segment_of(dir.path(), instance);
let records = crate::audit::writer::read_records(&path);
let mut forged = records[2].clone();
forged.seq += 100;
let mut bytes = std::fs::read(&path).unwrap();
bytes.extend_from_slice(&frame(&forged.to_json().unwrap()));
std::fs::write(&path, bytes).unwrap();
let report = verify(dir.path()).unwrap();
assert!(!report.holds());
}
#[test]
fn a_file_that_will_not_open_fails_the_log() {
let dir = tempfile::tempdir().unwrap();
write_session(dir.path(), 2);
let unopenable = dir.path().join(crate::audit::paths::day_file_name(
"2026-09-08".parse().unwrap(),
));
std::fs::create_dir(&unopenable).unwrap();
let report = verify(dir.path()).unwrap();
assert_eq!(report.unreadable.len(), 1);
assert!(!report.holds());
}
#[test]
fn an_empty_log_verifies() {
let dir = tempfile::tempdir().unwrap();
let report = verify(dir.path()).unwrap();
assert!(report.holds());
assert!(report.sessions.is_empty());
}
}