use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
const STATE_PENDING: &str = "pending";
const STATE_RETAINED: &str = "retained";
pub trait RecoveryHost: Send + Sync {
fn marker_path(&self) -> PathBuf;
fn now_ms(&self) -> u64;
fn authorized_scope(&self) -> Option<String>;
fn consent_epoch(&self) -> u64;
fn recovery_consent(&self) -> Option<RecoveryConsent> {
None
}
fn verify_replacement(&self) -> Result<u32, String>;
fn commit_recovery_if_authorized(
&self,
_receipt: &ReplayReceipt,
_consent: &RecoveryConsent,
_commit: &mut dyn FnMut() -> Result<(), String>,
) -> Result<bool, String> {
Ok(false)
}
fn event(&self, event: LedgerEvent, detail: &str);
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RecoveryConsent {
pub scope: String,
pub epoch: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct DurableConsentBinding {
v: u8,
epoch: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LedgerEvent {
Quarantined,
MarkerReadFailed,
MarkerUnreadable,
FinalizeRejected,
Completed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RecordedPair {
pub source: PathBuf,
pub destination: PathBuf,
}
impl RecordedPair {
pub fn into_paths(self) -> (PathBuf, PathBuf) {
(self.source, self.destination)
}
}
impl From<(PathBuf, PathBuf)> for RecordedPair {
fn from((source, destination): (PathBuf, PathBuf)) -> Self {
Self {
source,
destination,
}
}
}
pub fn verify_replacement_total<S, OpenError, QueryError>(
open: impl FnOnce() -> Result<S, OpenError>,
total: impl FnOnce(&S) -> Result<u32, QueryError>,
) -> Result<u32, String>
where
OpenError: std::fmt::Display,
QueryError: std::fmt::Display,
{
let store = open().map_err(|err| format!("post-replay store verification: {err}"))?;
total(&store).map_err(|err| format!("post-replay store query: {err}"))
}
#[derive(Debug, Clone)]
pub struct ReplayReceipt {
pub scope: String,
pub consent_generation: u64,
pub cursor_committed: bool,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct QuarantineMove {
#[serde(default)]
source: String,
#[serde(default)]
destination: String,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct RecoveryMarker {
v: u8,
#[serde(default)]
reason: String,
#[serde(default)]
created_at_ms: u64,
#[serde(default)]
bound_scope_digest: String,
#[serde(default)]
consent_generation: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
durable_consent: Option<DurableConsentBinding>,
#[serde(default = "default_state")]
state: String,
#[serde(default)]
quarantined: Vec<String>,
#[serde(default)]
pairs: Vec<QuarantineMove>,
#[serde(default)]
retained: Vec<String>,
}
fn default_state() -> String {
STATE_PENDING.to_string()
}
#[derive(Debug, Clone, Serialize)]
pub struct RecoveryStatus {
pub reason: String,
pub created_at_ms: u64,
pub state: String,
pub backups: Vec<RecoveryBackup>,
}
#[derive(Debug, Clone, Serialize)]
pub struct RecoveryBackup {
pub path: String,
pub bytes: Option<u64>,
}
fn load_marker(path: &Path) -> Option<RecoveryMarker> {
let bytes = std::fs::read(path).ok()?;
serde_json::from_slice::<RecoveryMarker>(&bytes)
.ok()
.filter(|marker| marker.v == 1)
}
fn scope_digest(scope: &str) -> String {
rightkit_fs::digest::sha256_bytes(scope.as_bytes())
}
pub struct RecoveryLedger<H: RecoveryHost> {
host: H,
lock: std::sync::Mutex<()>,
recorded: AtomicBool,
}
impl<H: RecoveryHost> RecoveryLedger<H> {
pub const fn new(host: H) -> Self {
Self {
host,
lock: std::sync::Mutex::new(()),
recorded: AtomicBool::new(false),
}
}
pub fn host(&self) -> &H {
&self.host
}
fn marker_path(&self) -> PathBuf {
self.host.marker_path()
}
fn lock(&self) -> Result<std::sync::MutexGuard<'_, ()>, String> {
self.lock
.lock()
.map_err(|_| "history recovery lock poisoned".to_string())
}
fn write_marker(&self, marker: &RecoveryMarker) -> Result<(), String> {
let path = self.marker_path();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|err| format!("create recovery marker dir: {err}"))?;
}
let bytes = serde_json::to_vec(marker)
.map_err(|err| format!("serialize recovery marker: {err}"))?;
let options = rightkit_fs::atomic::AtomicWriteOptions {
create_parents: true,
..Default::default()
};
rightkit_fs::atomic::atomic_write_with_options(&path, options, |file| {
std::io::Write::write_all(file, &bytes)
})
.map_err(|error| match error {
rightkit_fs::FsError::Io { source, .. }
| rightkit_fs::FsError::ReplacedButDirSyncFailed { source, .. } => {
format!("commit recovery marker: {source}")
}
other => format!("commit recovery marker: {other}"),
})
}
fn read_marker_strict(&self) -> Result<Option<RecoveryMarker>, String> {
let path = self.marker_path();
let bytes = match std::fs::read(&path) {
Ok(bytes) => bytes,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(format!("read recovery marker {}: {err}", path.display())),
};
match serde_json::from_slice::<RecoveryMarker>(&bytes) {
Ok(marker) if marker.v == 1 => Ok(Some(marker)),
_ => Err(format!(
"unreadable recovery marker {}: preserving all database files until the recorded transaction is resolved",
path.display()
)),
}
}
pub fn pending_pairs(&self) -> Result<Vec<RecordedPair>, String> {
let _guard = self.lock()?;
Ok(self
.read_marker_strict()?
.map(|marker| {
marker
.pairs
.into_iter()
.map(|pair| RecordedPair {
source: PathBuf::from(pair.source),
destination: PathBuf::from(pair.destination),
})
.collect()
})
.unwrap_or_default())
}
pub fn pending_path_pairs(&self) -> Result<Vec<(PathBuf, PathBuf)>, String> {
Ok(self
.pending_pairs()?
.into_iter()
.map(RecordedPair::into_paths)
.collect())
}
pub fn settle_path_pairs(
&self,
consumed: &[(PathBuf, PathBuf)],
landed: &[PathBuf],
) -> Result<(), String> {
let consumed: Vec<RecordedPair> = consumed
.iter()
.map(|(source, destination)| RecordedPair {
source: source.clone(),
destination: destination.clone(),
})
.collect();
self.settle_pairs(&consumed, landed)
}
pub fn settle_pairs(
&self,
consumed: &[RecordedPair],
landed: &[PathBuf],
) -> Result<(), String> {
let _guard = self.lock()?;
let Some(mut marker) = self.read_marker_strict()? else {
return Ok(());
};
marker.pairs.retain(|pair| {
!consumed.iter().any(|done| {
done.source == PathBuf::from(&pair.source)
&& done.destination == PathBuf::from(&pair.destination)
})
});
let ledger = if marker.state == STATE_RETAINED {
&mut marker.retained
} else {
&mut marker.quarantined
};
for destination in landed {
let destination = destination.to_string_lossy().into_owned();
if !ledger.contains(&destination) {
ledger.push(destination);
}
}
self.write_marker(&marker)
}
pub fn record_quarantine(
&self,
destinations: &[PathBuf],
reason: Option<&str>,
) -> Result<(), String> {
let consent = self.host.recovery_consent();
let _guard = self.lock()?;
let mut marker = self.read_marker_strict()?.unwrap_or_default();
let fresh: Vec<String> = destinations
.iter()
.map(|destination| destination.to_string_lossy().into_owned())
.filter(|path| {
!marker.quarantined.contains(path)
&& !marker.retained.contains(path)
&& !marker.pairs.iter().any(|pair| &pair.destination == path)
})
.collect();
if fresh.is_empty() {
return Ok(());
}
let stamp = self.host.now_ms();
marker.v = 1;
if let Some(reason) = reason {
marker.reason = reason.to_string();
}
marker.state = STATE_PENDING.to_string();
if marker.quarantined.is_empty() && marker.pairs.is_empty() {
marker.created_at_ms = stamp;
marker.consent_generation = self.host.consent_epoch();
marker.bound_scope_digest = consent
.as_ref()
.map(|consent| scope_digest(&consent.scope))
.or_else(|| {
self.host
.authorized_scope()
.map(|scope| scope_digest(&scope))
})
.unwrap_or_default();
marker.durable_consent = consent.map(|consent| DurableConsentBinding {
v: 1,
epoch: consent.epoch,
});
}
let recorded = fresh.len();
marker.quarantined.extend(fresh);
self.write_marker(&marker)?;
self.host.event(
LedgerEvent::Quarantined,
&format!(
"reason={} recorded={} listed={}",
marker.reason,
recorded,
marker.quarantined.len()
),
);
self.recorded.store(true, Ordering::Release);
Ok(())
}
pub fn take_recorded(&self) -> bool {
self.recorded.swap(false, Ordering::AcqRel)
}
pub fn status(&self) -> Option<RecoveryStatus> {
let marker = load_marker(&self.marker_path())?;
let mut paths: Vec<String> = Vec::new();
for path in marker
.quarantined
.iter()
.chain(marker.retained.iter())
.cloned()
.chain(marker.pairs.iter().map(|pair| pair.destination.clone()))
{
if !paths.contains(&path) {
paths.push(path);
}
}
let backups = paths
.into_iter()
.map(|path| RecoveryBackup {
bytes: std::fs::metadata(&path).ok().map(|meta| meta.len()),
path,
})
.collect();
Some(RecoveryStatus {
reason: marker.reason,
created_at_ms: marker.created_at_ms,
state: marker.state,
backups,
})
}
pub fn diagnostics(&self, label: &str) -> String {
let Some(status) = self.status() else {
return format!("{label}: no recovery event recorded.\n");
};
let mut out = format!("{label} diagnostics\n");
out.push_str(&format!("reason: {}\n", status.reason));
out.push_str(&format!("created_at_ms: {}\n", status.created_at_ms));
out.push_str(&format!("state: {}\n", status.state));
for backup in &status.backups {
match backup.bytes {
Some(bytes) => out.push_str(&format!("backup: {} ({bytes} bytes)\n", backup.path)),
None => out.push_str(&format!("backup: {} (missing)\n", backup.path)),
}
}
out
}
fn pending_marker(&self) -> Option<RecoveryMarker> {
let path = self.marker_path();
let bytes = match std::fs::read(&path) {
Ok(bytes) => bytes,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return None,
Err(err) => {
self.host
.event(LedgerEvent::MarkerReadFailed, &format!("{err}"));
return None;
}
};
match serde_json::from_slice::<RecoveryMarker>(&bytes) {
Ok(marker) if marker.v == 1 && marker.state == STATE_PENDING => Some(marker),
Ok(_) => None,
_ => {
self.host.event(
LedgerEvent::MarkerUnreadable,
"unreadable marker left in place",
);
None
}
}
}
pub fn pending_for_scope(&self, scope: &str) -> bool {
let Some(marker) = self.pending_marker() else {
return false;
};
!marker.bound_scope_digest.is_empty() && marker.bound_scope_digest == scope_digest(scope)
}
pub fn finish_after_replay(&self, receipt: &ReplayReceipt) -> Result<(), String> {
let Some(snapshot) = self.pending_marker() else {
return Ok(());
};
if !receipt.cursor_committed {
self.host
.event(LedgerEvent::FinalizeRejected, "reason=uncommitted");
return Ok(());
}
if snapshot.bound_scope_digest.is_empty()
|| snapshot.bound_scope_digest != scope_digest(&receipt.scope)
{
self.host.event(
LedgerEvent::FinalizeRejected,
"reason=scope_unbound_or_mismatch",
);
return Ok(());
}
let Some(binding) = snapshot
.durable_consent
.as_ref()
.filter(|binding| binding.v == 1)
else {
self.host.event(
LedgerEvent::FinalizeRejected,
"reason=durable_consent_unproven",
);
return Ok(());
};
if receipt.consent_generation != self.host.consent_epoch() {
self.host
.event(LedgerEvent::FinalizeRejected, "reason=consent_changed");
return Ok(());
}
let restored_total = self.host.verify_replacement()?;
let consent = RecoveryConsent {
scope: receipt.scope.clone(),
epoch: binding.epoch,
};
let mut commit = || self.commit_replay_snapshot(receipt, &snapshot, restored_total);
if !self
.host
.commit_recovery_if_authorized(receipt, &consent, &mut commit)?
{
self.host.event(
LedgerEvent::FinalizeRejected,
"reason=consent_commit_unavailable_or_changed",
);
}
Ok(())
}
fn commit_replay_snapshot(
&self,
receipt: &ReplayReceipt,
snapshot: &RecoveryMarker,
restored_total: u32,
) -> Result<(), String> {
let _guard = self.lock()?;
let Some(mut current) = self.read_marker_strict()? else {
self.host
.event(LedgerEvent::FinalizeRejected, "reason=marker_vanished");
return Ok(());
};
if current.state != STATE_PENDING
|| current.bound_scope_digest != snapshot.bound_scope_digest
|| current.consent_generation != snapshot.consent_generation
|| current
.durable_consent
.as_ref()
.map(|binding| (binding.v, binding.epoch))
!= snapshot
.durable_consent
.as_ref()
.map(|binding| (binding.v, binding.epoch))
{
self.host
.event(LedgerEvent::FinalizeRejected, "reason=marker_changed");
return Ok(());
}
if self
.host
.authorized_scope()
.map(|scope| scope_digest(&scope))
.as_deref()
!= Some(current.bound_scope_digest.as_str())
{
self.host.event(
LedgerEvent::FinalizeRejected,
"reason=scope_unbound_or_mismatch",
);
return Ok(());
}
if receipt.consent_generation != self.host.consent_epoch() {
self.host
.event(LedgerEvent::FinalizeRejected, "reason=consent_changed");
return Ok(());
}
let mut remaining = Vec::new();
let mut retained_now = 0usize;
for path in current.quarantined.drain(..) {
if snapshot.quarantined.contains(&path) {
current.retained.push(path);
retained_now += 1;
} else {
remaining.push(path);
}
}
current.quarantined = remaining;
current.state = if current.quarantined.is_empty() {
STATE_RETAINED
} else {
STATE_PENDING
}
.to_string();
self.write_marker(¤t)?;
self.host.event(
LedgerEvent::Completed,
&format!("retained_files={retained_now} verified_total={restored_total}"),
);
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicU64;
use std::sync::Mutex;
static NEXT: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Copy)]
enum DuringVerification {
Unchanged,
Revoke,
SwitchScope,
CorruptMarker,
Unsupported,
}
struct Host {
root: PathBuf,
epoch: AtomicU64,
durable_epoch: AtomicU64,
scope: Mutex<Option<String>>,
consent_commit: Mutex<()>,
during_verification: DuringVerification,
}
impl Host {
fn new(during_verification: DuringVerification) -> Self {
let root = std::env::temp_dir().join(format!(
"rightkit-recovery-marker-{}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir(&root).unwrap();
Self {
root,
epoch: AtomicU64::new(7),
durable_epoch: AtomicU64::new(41),
scope: Mutex::new(Some("owner".to_string())),
consent_commit: Mutex::new(()),
during_verification,
}
}
}
impl Drop for Host {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
impl RecoveryHost for Host {
fn marker_path(&self) -> PathBuf {
self.root.join("marker.json")
}
fn now_ms(&self) -> u64 {
100
}
fn authorized_scope(&self) -> Option<String> {
self.scope.lock().unwrap().clone()
}
fn consent_epoch(&self) -> u64 {
self.epoch.load(Ordering::Acquire)
}
fn recovery_consent(&self) -> Option<RecoveryConsent> {
Some(RecoveryConsent {
scope: self.authorized_scope()?,
epoch: self.durable_epoch.load(Ordering::Acquire),
})
}
fn verify_replacement(&self) -> Result<u32, String> {
match self.during_verification {
DuringVerification::Unchanged | DuringVerification::Unsupported => {}
DuringVerification::Revoke => {
self.epoch.fetch_add(1, Ordering::AcqRel);
}
DuringVerification::SwitchScope => {
*self.scope.lock().unwrap() = Some("other-owner".to_string());
}
DuringVerification::CorruptMarker => {
std::fs::write(self.marker_path(), b"broken marker").unwrap();
}
}
Ok(3)
}
fn commit_recovery_if_authorized(
&self,
receipt: &ReplayReceipt,
consent: &RecoveryConsent,
commit: &mut dyn FnMut() -> Result<(), String>,
) -> Result<bool, String> {
if matches!(self.during_verification, DuringVerification::Unsupported) {
return Ok(false);
}
let _guard = self.consent_commit.lock().unwrap();
if self.consent_epoch() != receipt.consent_generation
|| self.authorized_scope().as_deref() != Some(receipt.scope.as_str())
|| self.durable_epoch.load(Ordering::Acquire) != consent.epoch
{
return Ok(false);
}
commit()?;
Ok(true)
}
fn event(&self, _event: LedgerEvent, _detail: &str) {}
}
fn receipt() -> ReplayReceipt {
ReplayReceipt {
scope: "owner".to_string(),
consent_generation: 7,
cursor_committed: true,
}
}
#[test]
fn quarantine_preserves_invalid_marker_bytes() {
for bytes in [b"broken marker".as_slice(), b"{\"v\":2}"] {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
std::fs::write(ledger.marker_path(), bytes).unwrap();
assert!(ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.is_err());
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), bytes);
assert!(!ledger.take_recorded());
}
}
#[test]
fn quarantine_preserves_unreadable_marker() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
std::fs::create_dir(ledger.marker_path()).unwrap();
assert!(ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.is_err());
assert!(ledger.marker_path().is_dir());
assert!(!ledger.take_recorded());
}
#[test]
fn absent_marker_records_once_and_matching_replay_retains() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
let backup = ledger.host.root.join("backup");
ledger
.record_quarantine(std::slice::from_ref(&backup), None)
.unwrap();
assert!(ledger.take_recorded());
let before = std::fs::read(ledger.marker_path()).unwrap();
ledger
.record_quarantine(&[backup], Some("changed"))
.unwrap();
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), before);
assert!(!ledger.take_recorded());
ledger.finish_after_replay(&receipt()).unwrap();
let marker = ledger.read_marker_strict().unwrap().unwrap();
assert_eq!(marker.state, STATE_RETAINED);
assert!(marker.quarantined.is_empty());
assert_eq!(marker.retained.len(), 1);
}
#[test]
fn queued_pairs_keep_original_unbound_consent() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
let marker = RecoveryMarker {
v: 1,
state: STATE_PENDING.to_string(),
pairs: vec![QuarantineMove {
source: "old-store".to_string(),
destination: "old-backup".to_string(),
}],
..Default::default()
};
ledger.write_marker(&marker).unwrap();
let before = std::fs::read(ledger.marker_path()).unwrap();
ledger
.record_quarantine(&[PathBuf::from("old-backup")], None)
.unwrap();
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), before);
ledger
.record_quarantine(&[ledger.host.root.join("new-backup")], None)
.unwrap();
let current = ledger.read_marker_strict().unwrap().unwrap();
assert!(current.bound_scope_digest.is_empty());
assert_eq!(current.consent_generation, marker.consent_generation);
assert_eq!(current.pairs.len(), 1);
}
#[test]
fn consent_changes_during_verification_leave_pending_bytes_unchanged() {
for change in [DuringVerification::Revoke, DuringVerification::SwitchScope] {
let ledger = RecoveryLedger::new(Host::new(change));
ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.unwrap();
let before = std::fs::read(ledger.marker_path()).unwrap();
ledger.finish_after_replay(&receipt()).unwrap();
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), before);
}
}
#[test]
fn corruption_during_verification_is_reported_and_preserved() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::CorruptMarker));
ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.unwrap();
assert!(ledger.finish_after_replay(&receipt()).is_err());
assert_eq!(
std::fs::read(ledger.marker_path()).unwrap(),
b"broken marker"
);
}
#[test]
fn unsupported_consent_commit_keeps_recovery_pending() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unsupported));
ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.unwrap();
let before = std::fs::read(ledger.marker_path()).unwrap();
ledger.finish_after_replay(&receipt()).unwrap();
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), before);
}
#[test]
fn restart_uses_durable_epoch_instead_of_recorded_runtime_generation() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.unwrap();
let host = ledger.host;
host.epoch.store(0, Ordering::Release);
let restarted = RecoveryLedger::new(host);
let mut replay = receipt();
replay.consent_generation = 0;
restarted.finish_after_replay(&replay).unwrap();
assert_eq!(restarted.status().unwrap().state, STATE_RETAINED);
}
#[test]
fn restart_cannot_adopt_another_durable_epoch_with_same_runtime_generation() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
ledger
.record_quarantine(&[ledger.host.root.join("backup")], None)
.unwrap();
let before = std::fs::read(ledger.marker_path()).unwrap();
let host = ledger.host;
host.durable_epoch.fetch_add(1, Ordering::AcqRel);
let restarted = RecoveryLedger::new(host);
restarted.finish_after_replay(&receipt()).unwrap();
assert_eq!(std::fs::read(restarted.marker_path()).unwrap(), before);
}
#[test]
fn legacy_runtime_provenance_stays_readable_without_becoming_durable() {
let ledger = RecoveryLedger::new(Host::new(DuringVerification::Unchanged));
let marker = RecoveryMarker {
v: 1,
state: STATE_PENDING.to_string(),
bound_scope_digest: scope_digest("owner"),
consent_generation: 7,
quarantined: vec!["old-backup".to_string()],
..Default::default()
};
ledger.write_marker(&marker).unwrap();
let before = std::fs::read(ledger.marker_path()).unwrap();
assert_eq!(ledger.status().unwrap().state, STATE_PENDING);
ledger.finish_after_replay(&receipt()).unwrap();
assert_eq!(std::fs::read(ledger.marker_path()).unwrap(), before);
}
}