use super::*;
const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20;
fn page_zero_header(frames: &[(u64, Vec<u8>)]) -> Option<Header> {
frames
.iter()
.find(|(page_no, _)| *page_no == 0)
.and_then(|(_, bytes)| Header::from_page(bytes).ok())
}
fn sealed_under(h: &Header, dek: &[u8; crypto::KEY_LEN]) -> bool {
h.version >= VERSION_V9 && h.validate_seal(dek).is_ok()
}
fn scan_header_recovery_candidates(
file: &mut File,
secret: &[u8],
held_dek: Option<&[u8; crypto::KEY_LEN]>,
) -> Result<Option<Vec<wal::Frame>>> {
let mut best: Option<(u64, Vec<wal::Frame>)> = None;
for_each_wal_candidate(file, &mut |file, page_no| {
if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret, held_dek)? {
if best.as_ref().is_none_or(|(id, _)| txn_id > *id) {
best = Some((txn_id, frames));
}
}
Ok(())
})?;
Ok(best.map(|(_, f)| f))
}
fn for_each_wal_candidate(
file: &mut File,
f: &mut dyn FnMut(&mut File, u64) -> Result<()>,
) -> Result<()> {
let file_len = file.seek(SeekFrom::End(0))?;
if file_len < PAGE_SIZE as u64 * 2 {
return Ok(()); }
let page_sz = PAGE_SIZE as u64;
let mut offset = page_sz;
let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
while offset < file_len {
let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
chunk.resize(want, 0);
file.seek(SeekFrom::Start(offset))?;
let mut filled = 0;
while filled < want {
match file.read(&mut chunk[filled..])? {
0 => break,
n => filled += n,
}
}
if filled == 0 {
break;
}
let mut inner = 0usize;
while inner + 4 <= filled {
if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
let page_no = (offset + inner as u64) / page_sz;
f(file, page_no)?;
}
inner += PAGE_SIZE;
}
offset += filled as u64;
}
Ok(())
}
pub(crate) fn wal_header_salts(path: &Path) -> Result<Vec<([u8; crypto::SALT_LEN], KdfParams)>> {
let mut file = File::open(path)?;
let mut found: Vec<(u64, [u8; crypto::SALT_LEN], KdfParams)> = Vec::new();
for_each_wal_candidate(&mut file, &mut |file, page_no| {
if let Some((txn_id, _, h)) = committed_header_candidate(file, page_no)? {
found.push((
txn_id,
h.salt,
KdfParams {
m_cost: h.m_cost,
t_cost: h.t_cost,
p_cost: h.p_cost,
},
));
}
Ok(())
})?;
found.sort_by_key(|f| std::cmp::Reverse(f.0));
let mut out: Vec<([u8; crypto::SALT_LEN], KdfParams)> = Vec::new();
for (_, salt, kdf) in found {
if !out.iter().any(|(s, _)| *s == salt) {
out.push((salt, kdf));
}
}
Ok(out)
}
fn committed_header_candidate(
file: &mut File,
candidate_page_no: u64,
) -> Result<Option<(u64, Vec<wal::Frame>, Header)>> {
let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
return Ok(None);
};
let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
Some((_, b)) => b,
None => return Ok(None),
};
if header_bytes.len() != PAGE_SIZE {
return Ok(None);
}
let mut hbuf = [0u8; PAGE_SIZE];
hbuf.copy_from_slice(header_bytes);
let Ok(candidate) = Header::from_page(&hbuf) else {
return Ok(None);
};
if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
return Ok(None);
}
Ok(Some((txn_id, frames, candidate)))
}
fn try_recover_candidate(
file: &mut File,
candidate_page_no: u64,
secret: &[u8],
held_dek: Option<&[u8; crypto::KEY_LEN]>,
) -> Result<Option<(u64, Vec<wal::Frame>)>> {
let Some((txn_id, frames, candidate)) = committed_header_candidate(file, candidate_page_no)?
else {
return Ok(None);
};
if let Some(held) = held_dek {
if sealed_under(&candidate, held) {
return Ok(Some((txn_id, frames)));
}
}
let kdf = KdfParams {
m_cost: candidate.m_cost,
t_cost: candidate.t_cost,
p_cost: candidate.p_cost,
};
let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
Ok(k) => k,
Err(_) => return Ok(None),
};
let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
Ok(d) => d,
Err(_) => return Ok(None),
};
if candidate.validate_seal(&dek).is_err() {
return Ok(None);
}
Ok(Some((txn_id, frames)))
}
impl Mnemo {
pub fn create(
path: impl Into<PathBuf>,
passphrase: &str,
config: MnemoConfig,
) -> Result<Mnemo> {
Self::create_secret(path, passphrase.as_bytes(), SecretKind::Passphrase, config)
}
pub fn create_with_key(
path: impl Into<PathBuf>,
key: &[u8; crypto::KEY_LEN],
config: MnemoConfig,
) -> Result<Mnemo> {
let config = MnemoConfig {
kdf: KdfParams::raw_key(),
..config
};
Self::create_secret(path, key, SecretKind::Key, config)
}
pub fn generate_key() -> [u8; crypto::KEY_LEN] {
*crypto::random_dek()
}
pub(super) fn create_secret(
path: impl Into<PathBuf>,
secret: &[u8],
kind: SecretKind,
config: MnemoConfig,
) -> Result<Mnemo> {
Self::create_secret_as(path.into(), secret, kind, config, Create::Truncate, None)
}
pub(super) fn create_secret_new(
path: &Path,
secret: &[u8],
kind: SecretKind,
config: MnemoConfig,
salt: Option<[u8; crypto::SALT_LEN]>,
) -> Result<Mnemo> {
Self::create_secret_as(path.to_path_buf(), secret, kind, config, Create::New, salt)
}
pub(super) fn create_secret_as(
path: PathBuf,
secret: &[u8],
kind: SecretKind,
config: MnemoConfig,
create: Create,
salt: Option<[u8; crypto::SALT_LEN]>,
) -> Result<Mnemo> {
crypto::check_kdf_for(kind, config.kdf)?;
if config.dimensions == 0 {
return Err(MnemoError::Invalid("dimensions must be > 0".into()));
}
let version = resolve_format_version(config.format_version)?;
if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
return Err(MnemoError::Invalid(format!(
"a v9 file supports up to {} dimensions for now (one fp16 vector \
must fit a page); vector entries that span pages are not supported yet",
(PAYLOAD - veccol::PAGE_HEADER - 4) / 2
)));
}
let flags = if version >= VERSION_V9 && config.store_float_vectors {
FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
} else {
FLAG_ENCRYPTED
};
let file = open_with_lock(&path, true, create)?;
let salt = salt.unwrap_or_else(crypto::random_salt);
let dek = crypto::random_dek();
let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
let dek_nonce = crypto::random_nonce();
let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
if wrapped.len() != WRAPPED_DEK_LEN {
return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
}
let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
wrapped_dek.copy_from_slice(&wrapped);
let mut header = Header {
version,
page_size: PAGE_SIZE as u32,
flags,
dimensions: config.dimensions as u32,
created_at: memory::now_secs(),
write_counter: 0,
next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
catalog_start: 0,
catalog_pages: 0,
catalog_len: 0,
vector_count: 0,
m_cost: config.kdf.m_cost,
t_cost: config.kdf.t_cost,
p_cost: config.kdf.p_cost,
salt,
dek_nonce,
wrapped_dek,
index_start: 0,
index_pages: 0,
index_len: 0,
wal_start: 1,
wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
wal_seq: 0,
manifest_start: 0,
manifest_pages: 0,
manifest_len: 0,
seal_nonce: None,
seal_tag: None,
cache_start: 0,
cache_pages: 0,
cache_len: 0,
catalog_gen: 0,
index_gen: 0,
manifest_gen: 0,
cache_gen: 0,
free: RunPtr::default(),
keys: RunPtr::default(),
bm25: RunPtr::default(),
index_delta: RunPtr::default(),
catalog_delta: RunPtr::default(),
};
let mut pager = Pager::new(file, dek, 0, version);
let mut page = header.to_page();
header.apply_seal(&mut page, pager.dek())?;
pager.write_raw(0, &page)?;
pager.sync()?;
Ok(Mnemo {
pager,
header,
catalog: Vec::new(),
index: IdIndex::default(),
path: absolute(path),
dimensions: config.dimensions,
kdf: config.kdf,
dirty_catalog: false,
ann: None,
dirty_index: false,
index_persist: IndexPersist::default(),
ann_unloaded: false,
manifest: Vec::new(),
max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
{
V9_DEFAULT_MAX_SNAPSHOTS
} else {
config.max_snapshots
},
read_only: false,
cache_dir: Vec::new(),
cache_idx: DirectoryIndex::new(),
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
rekey_cleanup: None,
access_delta: None,
commit_point: None,
lock_note: None,
notices: Vec::new(),
window_committed: false,
shared: false,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
author: None,
presence_held: false,
clock_offset: 0,
legacy_coordination: None,
free: FreeList::default(),
pending_free: Vec::new(),
free_dirty: false,
key_slots: KeySlotRun::default(),
dirty_keys: false,
gen_leased: false,
col: VecColumn::default(),
recs: RecPages::default(),
bm25: Bm25State::default(),
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain: Vec::new(),
delta_rows: 0,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
Self::open_secret(path, passphrase.as_bytes()).map(passphrase_opened)
}
pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
Self::open_secret(path, key)
}
pub(super) fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
Self::open_secret_with(path, secret, false)
}
pub(super) fn open_secret_with(
path: impl Into<PathBuf>,
secret: &[u8],
allow_old: bool,
) -> Result<Mnemo> {
let path: PathBuf = path.into();
let file = open_with_lock(&path, true, Create::No)?;
Self::load_locked(path, file, secret, allow_old)
}
pub fn open_exclusive(
path: impl AsRef<Path>,
passphrase: &str,
op: &str,
opts: crate::SharedOptions,
) -> Result<Mnemo> {
Self::open_exclusive_secret(path.as_ref(), passphrase.as_bytes(), op, &opts, false)
.map(passphrase_opened)
}
pub fn open_exclusive_with_key(
path: impl AsRef<Path>,
key: &[u8; crypto::KEY_LEN],
op: &str,
opts: crate::SharedOptions,
) -> Result<Mnemo> {
Self::open_exclusive_secret(path.as_ref(), key, op, &opts, false)
}
pub(super) fn open_exclusive_secret(
path: &Path,
secret: &[u8],
op: &str,
opts: &crate::SharedOptions,
allow_old: bool,
) -> Result<Mnemo> {
let path = absolute(path.to_path_buf());
let (file, note) = lock_exclusive(&path, op, opts)?;
let mut db = Self::load_locked(path, file, secret, allow_old)?;
db.lock_note = note;
Ok(db)
}
pub(super) fn load_locked(
path: PathBuf,
file: LockedFile,
secret: &[u8],
allow_old: bool,
) -> Result<Mnemo> {
Self::load_locked_with(path, file, secret, None, allow_old)
}
pub(super) fn load_locked_with(
path: PathBuf,
mut file: LockedFile,
secret: &[u8],
held_dek: Option<&[u8; crypto::KEY_LEN]>,
allow_old: bool,
) -> Result<Mnemo> {
let mut hbuf = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
let mut header = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(e) => {
let scan = scan_header_recovery_candidates(&mut file, secret, held_dek)?;
let frames = scan.ok_or(e)?;
if !allow_old {
if let Some(h) = page_zero_header(&frames) {
if h.version < VERSION_V9 {
return Err(MnemoError::FormatTooOld { version: h.version });
}
}
}
for (page_no, bytes) in &frames {
if bytes.len() != PAGE_SIZE {
return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
}
file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
file.write_all(bytes)?;
}
file.sync_all()?;
crate::bump_fsync();
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
Header::from_page(&hbuf)?
}
};
let pending = wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)?;
let target = match pending.as_deref().and_then(page_zero_header) {
Some(h) => h,
None => header.clone(),
};
let kdf = KdfParams {
m_cost: target.m_cost,
t_cost: target.t_cost,
p_cost: target.p_cost,
};
let dek = match held_dek {
Some(held) if sealed_under(&target, held) => zeroize::Zeroizing::new(*held),
_ => {
let kek = crypto::derive_kek(secret, &target.salt, kdf)?;
crypto::unwrap_dek(&kek, &target.dek_nonce, &target.wrapped_dek)?
}
};
if !allow_old && target.version < VERSION_V9 {
return Err(MnemoError::FormatTooOld {
version: target.version,
});
}
if let Some(frames) = pending {
for (page_no, bytes) in &frames {
if bytes.len() != PAGE_SIZE {
return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
}
file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
file.write_all(bytes)?;
}
file.sync_all()?;
crate::bump_fsync();
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
header = Header::from_page(&hbuf)?;
}
Self::from_locked(path, file, header, dek, kdf, allow_old)
}
pub(super) fn from_locked(
path: PathBuf,
file: LockedFile,
mut header: Header,
dek: zeroize::Zeroizing<[u8; crypto::KEY_LEN]>,
kdf: KdfParams,
allow_old: bool,
) -> Result<Mnemo> {
header.validate_seal(&dek)?;
let on_disk_version = header.version;
if !allow_old && on_disk_version < VERSION_V9 {
return Err(MnemoError::FormatTooOld {
version: on_disk_version,
});
}
let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
let dimensions = header.dimensions as usize;
let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
(
load_catalog(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
header.catalog_gen,
)?,
false,
)
} else if on_disk_version == 4 {
let v4 = load_catalog_v4(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
)?;
(migrate_v4_catalog(&mut pager, v4)?, true)
} else {
return Err(MnemoError::UnsupportedVersion(on_disk_version));
};
let (catalog, delta_chain, delta_rows) =
apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
let index = build_id_index(&catalog);
let ann = if on_disk_version < 6 {
load_index(
&mut pager,
header.index_start,
header.index_pages,
header.index_len,
header.index_gen,
dimensions,
)?
} else {
None
};
let ann_unloaded = ann.is_none() && header.index_pages > 0;
let mut manifest = load_manifest(
&mut pager,
header.manifest_start,
header.manifest_pages,
header.manifest_len,
header.manifest_gen,
)?;
let migrated_pages_to_v6 = on_disk_version < 6;
if migrated_pages_to_v6 {
for e in &catalog {
if e.deleted {
continue;
}
for i in 0..e.page_count as u64 {
let payload = pager.read_page(e.start_page + i, 0)?;
pager.write_page(e.start_page + i, 0, &payload)?;
}
}
pager.set_version(6);
}
let migrated_header_to_v7 = on_disk_version < 7;
if migrated_header_to_v7 {
pager.set_version(7);
}
let migrated_header_to_v8 = on_disk_version < 8;
if migrated_header_to_v8 {
pager.set_version(VERSION);
}
let migrated = migrated_from_v4
|| migrated_pages_to_v6
|| migrated_header_to_v7
|| migrated_header_to_v8;
if migrated {
header.version = VERSION;
manifest.clear();
}
let dirty_index = migrated_pages_to_v6 && ann.is_some();
let cache_dir = load_cache_directory(
&mut pager,
header.cache_start,
header.cache_pages,
header.cache_len,
header.cache_gen,
)?;
let cache_idx = DirectoryIndex::build(&cache_dir);
let free = load_free_list(&mut pager, header.free)?;
let key_slots = load_key_slots(&mut pager, header.keys)?;
let col = VecColumn::from_catalog(&catalog);
let recs = RecPages::from_catalog(&catalog);
let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
let max_snapshots = if header.version >= VERSION_V9 {
V9_DEFAULT_MAX_SNAPSHOTS
} else {
DEFAULT_MAX_SNAPSHOTS
};
Ok(Mnemo {
pager,
header,
catalog,
index,
path: absolute(path),
dimensions,
kdf,
dirty_catalog: migrated,
ann,
dirty_index,
index_persist: IndexPersist {
forced: dirty_index,
..IndexPersist::default()
},
ann_unloaded,
manifest,
max_snapshots,
read_only: false,
cache_dir,
cache_idx,
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
rekey_cleanup: None,
access_delta: None,
commit_point: None,
lock_note: None,
notices: Vec::new(),
window_committed: false,
shared: false,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
author: None,
presence_held: false,
clock_offset: 0,
legacy_coordination: None,
free,
pending_free: Vec::new(),
free_dirty: false,
key_slots,
dirty_keys: false,
gen_leased: false,
col,
recs,
bm25: Bm25State {
manifest: bm25_manifest,
..Bm25State::default()
},
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain,
delta_rows,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: false,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
Self::open_read_only_secret(path, passphrase.as_bytes()).map(passphrase_opened)
}
pub fn open_read_only_with_key(
path: impl Into<PathBuf>,
key: &[u8; crypto::KEY_LEN],
) -> Result<Mnemo> {
Self::open_read_only_secret(path, key)
}
pub fn peek_dimensions(path: impl AsRef<std::path::Path>) -> Result<usize> {
let mut file = File::open(path.as_ref())?;
let mut hbuf = [0u8; PAGE_SIZE];
file.read_exact(&mut hbuf)?;
Ok(Header::from_page(&hbuf)?.dimensions as usize)
}
pub fn key_opens(
path: impl AsRef<std::path::Path>,
key: &[u8; crypto::KEY_LEN],
) -> Result<bool> {
let mut file = File::open(path.as_ref())?;
let mut hbuf = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
let header = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(parse_err) => {
return match scan_header_recovery_candidates(&mut file, key, None)? {
Some(_) => Ok(true),
None => Err(parse_err),
}
}
};
let kdf = KdfParams {
m_cost: header.m_cost,
t_cost: header.t_cost,
p_cost: header.p_cost,
};
let kek = crypto::derive_kek(key, &header.salt, kdf)?;
let dek = match crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek) {
Ok(dek) => dek,
Err(MnemoError::WrongPassphrase) => return Ok(false),
Err(e) => return Err(e),
};
header.validate_seal(&dek)?;
Ok(true)
}
pub(super) fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
let path: PathBuf = path.into();
let file = open_with_lock(&path, false, Create::No)?;
Self::read_only_from_locked(path, file, secret)
}
pub fn open_read_only_waiting(
path: impl AsRef<Path>,
passphrase: &str,
opts: crate::SharedOptions,
) -> Result<Mnemo> {
Self::open_read_only_waiting_secret(path.as_ref(), passphrase.as_bytes(), &opts)
.map(passphrase_opened)
}
pub fn open_read_only_waiting_with_key(
path: impl AsRef<Path>,
key: &[u8; crypto::KEY_LEN],
opts: crate::SharedOptions,
) -> Result<Mnemo> {
Self::open_read_only_waiting_secret(path.as_ref(), key, &opts)
}
fn open_read_only_waiting_secret(
path: &Path,
secret: &[u8],
opts: &crate::SharedOptions,
) -> Result<Mnemo> {
let path = absolute(path.to_path_buf());
let file = lock_shared(&path, opts)?;
Self::read_only_from_locked(path, file, secret)
}
fn read_only_from_locked(path: PathBuf, mut file: LockedFile, secret: &[u8]) -> Result<Mnemo> {
let mut hbuf = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
let header = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(parse_err) => {
let recoverable = scan_header_recovery_candidates(&mut file, secret, None)?;
return Err(match (parse_err, recoverable) {
(_, Some(frames))
if page_zero_header(&frames).is_some_and(|h| h.version < VERSION_V9) =>
{
MnemoError::NeedsWriteOpen {
reason: "torn header page in a pre-v9 file; run `mnemo compact \
<file>` (with `--key-file <key>` for a raw-key file), \
which recovers it and converts the file to v9",
}
}
(_, Some(_)) => MnemoError::NeedsWriteOpen {
reason: "torn header page needs recovery from WAL",
},
(e, None) => e,
});
}
};
if header.version < VERSION {
return Err(MnemoError::FormatTooOld {
version: header.version,
});
}
if wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)?
.is_some()
{
return Err(MnemoError::NeedsWriteOpen {
reason: if header.version < VERSION_V9 {
"pending WAL recovery in a pre-v9 file; run `mnemo compact <file>` \
(with `--key-file <key>` for a raw-key file), which recovers it and \
converts the file to v9"
} else {
"pending WAL recovery"
},
});
}
let kdf = KdfParams {
m_cost: header.m_cost,
t_cost: header.t_cost,
p_cost: header.p_cost,
};
let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
header.validate_seal(&dek)?;
let mut pager = Pager::new(file, dek, header.write_counter, header.version);
let dimensions = header.dimensions as usize;
let catalog = load_catalog(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
header.catalog_gen,
)?;
let (catalog, delta_chain, delta_rows) =
apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
let index = build_id_index(&catalog);
let ann: Option<IvfPqIndex> = None;
let ann_unloaded = header.index_pages > 0;
let manifest = load_manifest(
&mut pager,
header.manifest_start,
header.manifest_pages,
header.manifest_len,
header.manifest_gen,
)?;
let cache_dir = load_cache_directory(
&mut pager,
header.cache_start,
header.cache_pages,
header.cache_len,
header.cache_gen,
)?;
let cache_idx = DirectoryIndex::build(&cache_dir);
let free = load_free_list(&mut pager, header.free)?;
let key_slots = load_key_slots(&mut pager, header.keys)?;
let col = VecColumn::from_catalog(&catalog);
let recs = RecPages::from_catalog(&catalog);
let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
let max_snapshots = if header.version >= VERSION_V9 {
V9_DEFAULT_MAX_SNAPSHOTS
} else {
DEFAULT_MAX_SNAPSHOTS
};
Ok(Mnemo {
pager,
header,
catalog,
index,
path: absolute(path),
dimensions,
kdf,
dirty_catalog: false,
ann,
dirty_index: false,
index_persist: IndexPersist::default(),
ann_unloaded,
manifest,
max_snapshots,
read_only: true,
cache_dir,
cache_idx,
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
rekey_cleanup: None,
access_delta: None,
commit_point: None,
lock_note: None,
notices: Vec::new(),
window_committed: false,
shared: false,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
author: None,
presence_held: false,
clock_offset: 0,
legacy_coordination: None,
free,
pending_free: Vec::new(),
free_dirty: false,
key_slots,
dirty_keys: false,
gen_leased: false,
col,
recs,
bm25: Bm25State {
manifest: bm25_manifest,
..Bm25State::default()
},
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain,
delta_rows,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: false,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn set_author(&mut self, author: Option<String>) {
self.author = author;
}
pub fn author(&self) -> Option<&str> {
self.author.as_deref()
}
pub fn set_turn_compress_after(&mut self, secs: u64) {
self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
}
pub fn set_max_snapshots(&mut self, max: usize) {
self.max_snapshots = max;
self.snapshot_cap_explicit = true;
}
pub(crate) fn header_salt(&self) -> [u8; crypto::SALT_LEN] {
self.header.salt
}
pub(crate) fn disk_salt(&mut self) -> Result<[u8; crypto::SALT_LEN]> {
let page = self.pager.read_raw(0)?;
Ok(Header::from_page(&page)?.salt)
}
pub fn notices(&self) -> &[String] {
&self.notices
}
pub fn dimensions(&self) -> usize {
self.dimensions
}
pub fn len(&self) -> usize {
self.catalog.iter().filter(|e| !e.deleted).count()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
pub(crate) fn passphrase_floor_notice(path: &Path, kdf: KdfParams) -> Option<String> {
kdf.below_passphrase_floor().then(|| {
format!(
"{} is protected by a passphrase on raw-key KDF parameters ({} KiB, {} pass), \
which makes each guess of it cheap; `mnemo rekey` or `mnemo compact` raises \
the parameters",
path.display(),
kdf.m_cost,
kdf.t_cost
)
})
}
fn passphrase_opened(mut db: Mnemo) -> Mnemo {
if let Some(n) = passphrase_floor_notice(&db.path, db.kdf) {
db.notices.push(n);
}
db
}