use std::collections::HashMap;
use std::sync::Arc;
use zeroize::Zeroizing;
use super::*;
use crate::shared::{BrainLocks, Call, CancelToken, LockKind, SharedOptions};
#[derive(Clone, Debug, Default)]
pub(crate) struct AccessDelta {
memories: HashMap<u128, (u32, i64)>,
cache: HashMap<(String, [u8; 32]), (u32, i64)>,
}
impl AccessDelta {
pub(crate) fn memory_hit(&mut self, id: u128, at: i64) {
let e = self.memories.entry(id).or_insert((0, at));
e.0 = e.0.saturating_add(1);
e.1 = e.1.max(at);
}
pub(crate) fn cache_hit(&mut self, namespace: &str, key_hash: [u8; 32], at: i64) {
let e = self
.cache
.entry((namespace.to_string(), key_hash))
.or_insert((0, at));
e.0 = e.0.saturating_add(1);
e.1 = e.1.max(at);
}
}
impl Mnemo {
pub fn open_shared(
path: impl AsRef<Path>,
passphrase: &str,
opts: SharedOptions,
) -> Result<SharedMnemo> {
SharedMnemo::open(path.as_ref(), passphrase.as_bytes(), opts)
}
pub fn open_shared_with_key(
path: impl AsRef<Path>,
key: &[u8; crypto::KEY_LEN],
opts: SharedOptions,
) -> Result<SharedMnemo> {
SharedMnemo::open(path.as_ref(), key, opts)
}
fn apply_access_delta(&mut self, d: &AccessDelta) {
for (id, (n, at)) in &d.memories {
if let Some(idx) = self.index.get(*id, &self.catalog) {
self.touch(idx);
let e = &mut self.catalog[idx];
e.access_count = e.access_count.saturating_add(*n);
e.accessed_at = e.accessed_at.max(*at);
self.dirty_catalog = true;
}
}
for ((ns, key), (n, at)) in &d.cache {
for e in self.cache_dir.iter_mut() {
if !e.deleted && e.namespace == *ns && e.key_hash == *key {
e.access_count = e.access_count.saturating_add(*n);
e.accessed_at = e.accessed_at.max(*at);
self.dirty_cache = true;
}
}
}
}
}
#[derive(PartialEq, Debug)]
struct FileSettings {
max_snapshots: usize,
snapshot_cap_explicit: bool,
turn_compress_after: i64,
cache_budgets: Vec<(String, usize, u64)>,
cache_flush_policy: String,
}
impl FileSettings {
fn of(db: &Mnemo) -> Self {
FileSettings {
max_snapshots: db.max_snapshots,
snapshot_cap_explicit: db.snapshot_cap_explicit,
turn_compress_after: db.turn_compress_after,
cache_budgets: {
let mut b: Vec<_> = db
.cache_budgets
.iter()
.map(|(ns, b)| (ns.clone(), b.max_entries, b.max_bytes))
.collect();
b.sort();
b
},
cache_flush_policy: format!("{:?}", db.cache_flush_policy),
}
}
}
const SETTINGS_KEPT: &str = "a shared-mode handle keeps the file's default settings in v0.14 \
(snapshot retention, cache budgets, the cache flush policy, turn compression): every \
process's commits would apply one handle's setting to the whole file";
fn settings_refused() -> MnemoError {
MnemoError::Unsupported(format!("{SETTINGS_KEPT}. Nothing was changed"))
}
fn part_stored(e: impl std::fmt::Display) -> MnemoError {
MnemoError::MaybeStored(format!(
"{e}; an earlier commit in this window is in the file, so part of the window \
was stored"
))
}
const HEAL_NEEDS_WRITE: &str = "the file has a pending WAL or a torn header to recover, \
and this process can't write to it";
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
struct Stamp {
identity: Option<(u64, u64)>,
seal: Option<([u8; crypto::NONCE_LEN], [u8; crypto::TAG_LEN])>,
}
impl Stamp {
fn of(file: &File, header: &Header) -> Self {
Stamp {
identity: identity(file),
seal: header.seal_nonce.zip(header.seal_tag),
}
}
}
#[cfg(unix)]
fn identity(file: &File) -> Option<(u64, u64)> {
use std::os::unix::fs::MetadataExt;
file.metadata().ok().map(|m| (m.dev(), m.ino()))
}
#[cfg(not(unix))]
fn identity(_file: &File) -> Option<(u64, u64)> {
None
}
fn read_page0(file: &mut File) -> std::io::Result<[u8; PAGE_SIZE]> {
let mut page = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut page)?;
Ok(page)
}
enum Next {
Ready,
NeedExclusive,
}
pub struct SharedMnemo {
locks: BrainLocks,
opts: SharedOptions,
path: PathBuf,
secret: Zeroizing<Vec<u8>>,
db: Option<Mnemo>,
stamp: Option<Stamp>,
delta: AccessDelta,
embedder: Option<Arc<dyn Embedder>>,
reloads: u64,
rekey_cleanup: Option<String>,
}
impl std::fmt::Debug for SharedMnemo {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SharedMnemo")
.field("path", &self.path)
.field("loaded", &self.db.is_some())
.field("reloads", &self.reloads)
.finish()
}
}
impl SharedMnemo {
fn open(path: &Path, secret: &[u8], opts: SharedOptions) -> Result<SharedMnemo> {
let locks = BrainLocks::new(path, &opts)?;
let mut s = SharedMnemo {
locks,
opts,
path: absolute(path.to_path_buf()),
secret: Zeroizing::new(secret.to_vec()),
db: None,
stamp: None,
delta: AccessDelta::default(),
embedder: None,
reloads: 0,
rekey_cleanup: None,
};
s.read(|_| Ok(()))?;
Ok(s)
}
pub fn read<R>(&mut self, f: impl FnOnce(&mut Mnemo) -> Result<R>) -> Result<R> {
self.window(LockKind::Shared, |s| s.run_read(f))
}
pub fn write<R>(&mut self, f: impl FnOnce(&mut Mnemo) -> Result<R>) -> Result<R> {
self.window(LockKind::Exclusive, |s| s.run_write(f))
}
pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
let identity = embedder.identity();
let needs_write = self.read(move |db| {
if db.embedder_would_record(&identity)? {
return Ok(Some(embedder));
}
db.set_embedder(embedder)?;
Ok(None)
})?;
if let Some(embedder) = needs_write {
self.write(|db| db.set_embedder(embedder))?;
}
Ok(())
}
pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
self.rekey_secret(new_passphrase.as_bytes(), kdf)
}
pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
self.rekey_secret(new_key, KdfParams::raw_key())
}
pub fn rekey_cleanup_error(&self) -> Option<&str> {
self.rekey_cleanup.as_deref()
}
fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
let held_back = std::mem::take(&mut self.delta);
let mut in_force = None;
let r = self.write(|db| {
db.rekey_unchecked(secret, kdf)?;
in_force = Some(db.rekey_cleanup.clone());
Ok(())
});
self.delta = held_back;
if let Some(cleanup) = in_force {
self.secret = Zeroizing::new(secret.to_vec());
self.rekey_cleanup = cleanup;
}
r
}
pub fn set_cancel(&mut self, token: Option<CancelToken>) {
self.opts.cancel = token;
}
pub fn embedder_arc(&self) -> Option<Arc<dyn Embedder>> {
self.embedder.clone()
}
pub fn notices(&self) -> &[String] {
self.locks.notices()
}
pub fn options(&self) -> &SharedOptions {
&self.opts
}
pub fn close(self) -> Result<()> {
Ok(())
}
#[cfg(any(test, feature = "failpoints"))]
#[doc(hidden)]
pub fn __reloads(&self) -> u64 {
self.reloads
}
#[cfg(any(test, feature = "failpoints"))]
#[doc(hidden)]
pub fn __page_cache(&self) -> (usize, usize) {
self.db.as_ref().map_or((0, 0), |d| d.pager.cache_stats())
}
#[cfg(any(test, feature = "failpoints"))]
#[doc(hidden)]
pub fn __state(&mut self) -> Option<&mut Mnemo> {
self.db.as_mut()
}
fn window<R>(&mut self, kind: LockKind, run: impl FnOnce(&mut Self) -> Result<R>) -> Result<R> {
let mut call = Call::new(self.opts.deadline, self.opts.cancel.as_ref());
let mut lock = kind;
let held = loop {
let mut held = match self.locks.acquire(lock, &mut call) {
Ok(h) => h,
Err(MnemoError::Io(e))
if lock != kind && e.kind() == std::io::ErrorKind::PermissionDenied =>
{
return Err(MnemoError::NeedsWriteOpen {
reason: HEAL_NEEDS_WRITE,
})
}
Err(e) => return Err(e),
};
let file = held.take_file();
match self.catch_up(lock, file)? {
Next::Ready => break held,
Next::NeedExclusive => lock = LockKind::Exclusive,
}
};
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(self)));
drop(held);
if let Some(db) = self.db.as_mut() {
drop(db.pager.detach_file());
self.embedder = db.embedder.clone();
}
match r {
Ok(r) => r,
Err(panic) => {
self.stamp = None;
std::panic::resume_unwind(panic)
}
}
}
fn catch_up(&mut self, lock: LockKind, mut file: LockedFile) -> Result<Next> {
let header = match read_page0(&mut file)
.map_err(MnemoError::Io)
.and_then(|p| Header::from_page(&p))
{
Ok(h) => h,
Err(_) if lock == LockKind::Shared => return Ok(Next::NeedExclusive),
Err(_) => return self.load(file),
};
let pending = wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)?
.is_some();
if pending {
if lock == LockKind::Shared {
return Ok(Next::NeedExclusive);
}
return self.load(file);
}
let stamp = Stamp::of(&file, &header);
if let (Some(db), Some(held)) = (self.db.as_mut(), self.stamp) {
if held == stamp {
db.pager.attach_file(file);
return Ok(Next::Ready);
}
}
if let Some(old) = &self.db {
if header.validate_seal(old.pager.dek()).is_ok() {
let dek = Zeroizing::new(*old.pager.dek());
let kdf = KdfParams {
m_cost: header.m_cost,
t_cost: header.t_cost,
p_cost: header.p_cost,
};
let new = Mnemo::from_locked(self.path.clone(), file, header, dek, kdf, false)?;
self.install(new);
return Ok(Next::Ready);
}
}
self.load(file)
}
fn load(&mut self, file: LockedFile) -> Result<Next> {
let new = self.reopen(file)?;
self.install(new);
Ok(Next::Ready)
}
fn reopen(&self, file: LockedFile) -> Result<Mnemo> {
let held = self.db.as_ref().map(|d| Zeroizing::new(*d.pager.dek()));
Mnemo::load_locked_with(
self.path.clone(),
file,
&self.secret,
held.as_deref(),
false,
)
}
fn install(&mut self, mut new: Mnemo) {
if let Some(old) = &self.db {
new.pager.set_cache_capacity(old.pager.cache_stats().1);
new.set_vector_cache_budget(old.vector_cache_budget);
}
if let Some(e) = &self.embedder {
new.embedder = Some(e.clone());
}
new.shared = true;
self.stamp = Some(Stamp::of(new.pager.file(), &new.header));
self.db = Some(new);
self.reloads += 1;
}
fn run_read<R>(&mut self, f: impl FnOnce(&mut Mnemo) -> Result<R>) -> Result<R> {
let db = self.db.as_mut().expect("loaded by the window");
db.read_only = true;
db.access_delta = Some(std::mem::take(&mut self.delta));
let before = FileSettings::of(db);
let r = f(db);
db.read_only = false;
self.delta = db.access_delta.take().unwrap_or_default();
if FileSettings::of(db) != before {
self.stamp = None;
return Err(settings_refused());
}
r
}
fn run_write<R>(&mut self, f: impl FnOnce(&mut Mnemo) -> Result<R>) -> Result<R> {
self.stamp = None;
let db = self.db.as_mut().expect("loaded by the window");
db.read_only = false;
db.access_delta = None;
db.commit_point = None;
db.window_committed = false;
db.gen_leased = false;
let delta = std::mem::take(&mut self.delta);
db.apply_access_delta(&delta);
let before = FileSettings::of(db);
let r = f(db).map_err(in_shared_window);
let db = self.db.as_mut().expect("loaded by the window");
let r = match r {
Ok(r) => r,
Err(e) if db.window_committed => return Err(part_stored(e)),
Err(e) if db.commit_point.is_some() => {
return Err(MnemoError::MaybeStored(e.to_string()))
}
Err(e) => {
self.delta = delta;
return Err(e);
}
};
if FileSettings::of(db) != before {
if db.window_committed || db.commit_point.is_some() {
return Err(part_stored(format!(
"{SETTINGS_KEPT}, so the setting wasn't applied"
)));
}
self.delta = delta;
return Err(settings_refused());
}
let committed_before = db.window_committed;
#[cfg(any(test, feature = "failpoints"))]
crate::__failpoints::run_before_shared_commit();
if let Err(e) = db.flush().map_err(in_shared_window) {
match self.finish_failed_commit(e) {
Ok(()) => {}
Err(e @ MnemoError::MaybeStored(_)) => return Err(e),
Err(e) if committed_before => return Err(part_stored(e)),
Err(e) => return Err(e),
}
}
#[cfg(any(test, feature = "failpoints"))]
crate::__failpoints::run_after_shared_commit();
let db = self.db.as_ref().expect("loaded");
if !is_file_at(db.pager.file(), &db.path)? {
return Err(MnemoError::ReplacedDuringWrite {
path: db.path.clone(),
});
}
self.stamp = Some(Stamp::of(db.pager.file(), &db.header));
Ok(r)
}
fn finish_failed_commit(&mut self, e: MnemoError) -> Result<()> {
let db = self.db.as_mut().expect("loaded");
let Some(txn) = db.commit_point.take() else {
return Err(e);
};
let Some(mut file) = db.pager.detach_file() else {
return Err(MnemoError::MaybeStored(e.to_string()));
};
let replay = (|| -> Result<()> {
let Ok(header) = read_page0(&mut file)
.map_err(MnemoError::Io)
.and_then(|p| Header::from_page(&p))
else {
return Ok(());
};
if let Some(frames) = wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)? {
for (page_no, bytes) in &frames {
crate::pager::__failpoint_check()?;
file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
file.write_all(bytes)?;
}
file.sync_all()?;
crate::bump_fsync();
}
Ok(())
})();
if let Err(e2) = replay {
return Err(MnemoError::MaybeStored(format!(
"{e}; finishing the commit failed too: {e2}"
)));
}
match self.reopen(file) {
Ok(new) => {
let stored = new.header.wal_seq >= txn;
self.install(new);
if stored {
Ok(())
} else {
self.stamp = None;
Err(e)
}
}
Err(e2) => Err(MnemoError::MaybeStored(format!(
"{e}; reading the file back failed: {e2}"
))),
}
}
}
fn in_shared_window(e: MnemoError) -> MnemoError {
match e {
MnemoError::Replaced { path, missing, .. } => MnemoError::Replaced {
path,
shared: true,
missing,
},
e => e,
}
}