use super::*;
pub(super) fn open_with_lock(path: &Path, write: bool, create: Create) -> Result<LockedFile> {
const ATTEMPTS: usize = 8;
for _ in 0..ATTEMPTS {
let file = open_and_lock_once(path, write, create)?;
if is_file_at(&file, path)? {
if create == Create::Truncate {
file.set_len(0)?;
}
return Ok(file);
}
if create == Create::New {
return Err(MnemoError::Replaced {
path: path.to_path_buf(),
shared: false,
missing: false,
});
}
}
Err(MnemoError::Replaced {
path: path.to_path_buf(),
shared: false,
missing: false,
})
}
pub(super) fn lock_exclusive(
path: &Path,
op: &str,
opts: &crate::SharedOptions,
) -> Result<(LockedFile, Option<crate::shared::Held>)> {
use crate::shared::{BrainLocks, Call, LockKind};
let mut locks = match BrainLocks::for_plain(path, opts) {
Ok(l) => l,
Err(MnemoError::Unsupported(_)) => {
return Ok((open_with_lock(path, true, Create::No)?, None));
}
Err(e) => return Err(e),
};
let mut call = Call::new(opts.deadline, opts.cancel.as_ref());
let mut held = locks.acquire(LockKind::Exclusive, &mut call)?;
held.announce(op);
let file = held.take_file();
Ok((file, Some(held)))
}
pub(super) fn lock_shared(path: &Path, opts: &crate::SharedOptions) -> Result<LockedFile> {
use crate::shared::{BrainLocks, Call, LockKind};
let mut locks = match BrainLocks::for_plain(path, opts) {
Ok(l) => l,
Err(MnemoError::Unsupported(_)) => return open_with_lock(path, false, Create::No),
Err(e) => return Err(e),
};
let mut call = Call::new(opts.deadline, opts.cancel.as_ref());
let mut held = locks.acquire(LockKind::Shared, &mut call)?;
Ok(held.take_file())
}
pub(super) fn absolute(path: PathBuf) -> PathBuf {
if path.is_absolute() {
return path;
}
match std::env::current_dir() {
Ok(dir) => dir.join(path),
Err(_) => path,
}
}
impl Mnemo {
pub(super) fn ensure_not_replaced(&self) -> Result<()> {
let at = match file_at(self.pager.file(), &self.path) {
Ok(at) => at,
Err(_) if !self.shared => return Ok(()),
Err(e) => return Err(e),
};
match at {
AtPath::Same => Ok(()),
AtPath::Other | AtPath::Nothing => Err(MnemoError::Replaced {
path: self.path.clone(),
shared: false,
missing: at == AtPath::Nothing,
}),
}
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum AtPath {
Same,
Other,
Nothing,
}
pub(crate) fn is_file_at(file: &File, path: &Path) -> Result<bool> {
Ok(file_at(file, path)? == AtPath::Same)
}
pub(crate) fn file_at(file: &File, path: &Path) -> Result<AtPath> {
#[cfg(any(test, feature = "failpoints"))]
if crate::__failpoints::take_path_check_failure() {
return Err(MnemoError::Io(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"path check failed (failpoint)",
)));
}
let at_path = match path_identity(path) {
Ok(id) => id,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(AtPath::Nothing),
Err(e) => return Err(MnemoError::Io(e)),
};
if file_identity(file)? == at_path {
Ok(AtPath::Same)
} else {
Ok(AtPath::Other)
}
}
#[cfg(unix)]
fn file_identity(file: &File) -> std::io::Result<(u64, u64)> {
use std::os::unix::fs::MetadataExt;
let m = file.metadata()?;
Ok((m.dev(), m.ino()))
}
#[cfg(unix)]
fn path_identity(path: &Path) -> std::io::Result<(u64, u64)> {
use std::os::unix::fs::MetadataExt;
let m = std::fs::metadata(path)?;
Ok((m.dev(), m.ino()))
}
#[cfg(not(unix))]
fn file_identity(file: &File) -> std::io::Result<same_file::Handle> {
same_file::Handle::from_file(file.try_clone()?)
}
#[cfg(not(unix))]
fn path_identity(path: &Path) -> std::io::Result<same_file::Handle> {
same_file::Handle::from_path(path)
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(super) enum Create {
No,
Truncate,
New,
}
fn open_and_lock_once(path: &Path, write: bool, create: Create) -> Result<LockedFile> {
let mut opts = OpenOptions::new();
opts.read(true);
if write {
opts.write(true);
}
match create {
Create::No => {}
Create::Truncate => {
opts.create(true);
}
Create::New => {
opts.create_new(true);
}
}
owner_only(&mut opts);
let file = opts.open(path)?;
let result = if write {
FileExt::try_lock_exclusive(&file)
} else {
FileExt::try_lock_shared(&file)
};
match result {
Ok(()) => Ok(LockedFile::new(file)),
Err(e) if is_lock_conflict(&e) => Err(MnemoError::Locked {
path: path.to_path_buf(),
holder: None,
}),
Err(e) => Err(MnemoError::Io(e)),
}
}
pub(super) fn owner_only(opts: &mut OpenOptions) -> &mut OpenOptions {
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
opts
}
pub(super) fn copy_mode(from: &File, to: &File) -> Result<()> {
#[cfg(unix)]
to.set_permissions(from.metadata()?.permissions())?;
#[cfg(not(unix))]
let _ = (from, to);
Ok(())
}
pub(super) fn paths_refer_to_same_file(a: &Path, b: &Path) -> Option<bool> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let ma = std::fs::metadata(a).ok()?;
let mb = std::fs::metadata(b).ok()?;
Some(ma.dev() == mb.dev() && ma.ino() == mb.ino())
}
#[cfg(not(unix))]
{
let ca = std::fs::canonicalize(a).ok()?;
let cb = std::fs::canonicalize(b).ok()?;
Some(ca == cb)
}
}
pub(super) fn temp_sibling(dest: &Path, what: &str) -> PathBuf {
#[cfg(any(test, feature = "failpoints"))]
if let Some(planted) = crate::__failpoints::take_next_temp_name() {
return planted;
}
use std::sync::atomic::{AtomicU64, Ordering};
static NEXT: AtomicU64 = AtomicU64::new(0);
let n = NEXT.fetch_add(1, Ordering::Relaxed);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_nanos())
.unwrap_or(0);
let mut buf = dest.as_os_str().to_os_string();
buf.push(format!(
".{what}-{}-{n}-{nanos:08x}.tmp",
std::process::id()
));
PathBuf::from(buf)
}
pub(super) const TEMP_NAME_ATTEMPTS: usize = 16;
pub(super) fn is_name_taken(e: &MnemoError) -> bool {
matches!(e, MnemoError::Io(io) if io.kind() == std::io::ErrorKind::AlreadyExists)
}
pub(crate) fn is_lock_conflict(e: &std::io::Error) -> bool {
if e.kind() == std::io::ErrorKind::WouldBlock {
return true;
}
#[cfg(windows)]
if let Some(code) = e.raw_os_error() {
if code == 33 || code == 32 {
return true;
}
}
false
}
pub(super) fn read_run_bytes(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<u8>> {
if pages < RUN_CHUNK_PAGES {
let mut buf = Vec::with_capacity(len as usize);
for i in 0..pages {
buf.extend_from_slice(&pager.read_page(start + i, gen)?);
}
buf.truncate(len as usize);
return Ok(buf);
}
let err = std::cell::RefCell::new(None);
let mut buf = Vec::with_capacity(len as usize);
let res = RunReader::new(pager, &err, start, pages, len, gen).read_to_end(&mut buf);
if let Some(e) = err.into_inner() {
return Err(e);
}
res?;
Ok(buf)
}
const RUN_CHUNK_PAGES: u64 = 256;
pub(super) struct RunReader<'a> {
pub(super) pager: &'a mut Pager,
pub(super) err: &'a std::cell::RefCell<Option<MnemoError>>,
next_page: u64,
end_page: u64,
gen: u64,
remaining: u64,
images: Vec<u8>,
pages_in: usize,
page_i: usize,
pos: usize,
lim: usize,
}
impl<'a> RunReader<'a> {
pub(super) fn new(
pager: &'a mut Pager,
err: &'a std::cell::RefCell<Option<MnemoError>>,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Self {
RunReader {
pager,
err,
next_page: start,
end_page: start + pages,
gen,
remaining: len,
images: Vec::new(),
pages_in: 0,
page_i: 0,
pos: 0,
lim: 0,
}
}
fn advance(&mut self) -> std::io::Result<bool> {
if self.remaining == 0 {
return Ok(false);
}
if self.page_i + 1 < self.pages_in {
self.page_i += 1;
} else {
if self.next_page >= self.end_page {
return Ok(false);
}
let n = (self.end_page - self.next_page).min(RUN_CHUNK_PAGES);
let want: Vec<(u64, u64)> = (self.next_page..self.next_page + n)
.map(|p| (p, self.gen))
.collect();
self.images = Vec::new();
match self.pager.read_pages_uncached(&want) {
Ok(b) => {
self.pages_in = b.len();
self.images = b.into_images();
}
Err(e) => {
let msg = e.to_string();
*self.err.borrow_mut() = Some(e);
return Err(std::io::Error::other(msg));
}
}
self.next_page += n;
self.page_i = 0;
}
let avail = (self.remaining as usize).min(PAYLOAD);
self.remaining -= avail as u64;
self.pos = self.page_i * PAGE_SIZE + crate::crypto::NONCE_LEN;
self.lim = self.pos + avail;
Ok(true)
}
}
impl Read for RunReader<'_> {
#[inline]
fn read_exact(&mut self, out: &mut [u8]) -> std::io::Result<()> {
if self.lim - self.pos >= out.len() {
out.copy_from_slice(&self.images[self.pos..self.pos + out.len()]);
self.pos += out.len();
return Ok(());
}
let mut done = 0;
while done < out.len() {
match self.read(&mut out[done..])? {
0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
n => done += n,
}
}
Ok(())
}
#[inline]
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
while self.pos == self.lim {
if !self.advance()? {
return Ok(0);
}
}
let n = out.len().min(self.lim - self.pos);
out[..n].copy_from_slice(&self.images[self.pos..self.pos + n]);
self.pos += n;
Ok(n)
}
}
pub(super) struct ExactVec<T> {
pub(super) cap: usize,
pub(super) _t: std::marker::PhantomData<T>,
}
impl<'de, T: Deserialize<'de>> serde::de::DeserializeSeed<'de> for ExactVec<T> {
type Value = Vec<T>;
fn deserialize<D: serde::Deserializer<'de>>(
self,
d: D,
) -> std::result::Result<Vec<T>, D::Error> {
d.deserialize_seq(self)
}
}
impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for ExactVec<T> {
type Value = Vec<T>;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a sequence")
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(
self,
mut seq: A,
) -> std::result::Result<Vec<T>, A::Error> {
let mut v = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(self.cap));
while let Some(x) = seq.next_element()? {
v.push(x);
}
Ok(v)
}
}
pub(super) fn load_catalog(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<CatalogEntry>> {
if pages == 0 {
return Ok(Vec::new());
}
let cap = (len / 8) as usize;
if pager.version() < VERSION_V9 {
let buf = read_run_bytes(pager, start, pages, len, gen)?;
let mut de = rmp_serde::Deserializer::from_read_ref(&buf);
return serde::de::DeserializeSeed::deserialize(
ExactVec::<CatalogEntry> {
cap,
_t: std::marker::PhantomData,
},
&mut de,
)
.map_err(|e| MnemoError::Serialize(e.to_string()));
}
let err = std::cell::RefCell::new(None);
let mut de = rmp_serde::Deserializer::new(RunReader::new(pager, &err, start, pages, len, gen));
let res = serde::de::DeserializeSeed::deserialize(
ExactVec::<CatalogEntry> {
cap,
_t: std::marker::PhantomData,
},
&mut de,
);
drop(de);
match (res, err.into_inner()) {
(Ok(v), _) => Ok(v),
(Err(_), Some(page_err)) => Err(page_err),
(Err(e), None) => Err(MnemoError::Serialize(e.to_string())),
}
}
#[derive(Serialize, Deserialize)]
pub(super) struct CatalogDelta {
pub(super) prev: (u64, u64, u64, u64),
pub(super) rows: Vec<(u32, CatalogEntry)>,
}
pub(super) fn ptr_tuple(p: RunPtr) -> (u64, u64, u64, u64) {
(p.start, p.pages, p.len, p.gen)
}
pub(super) fn tuple_ptr(t: (u64, u64, u64, u64)) -> RunPtr {
RunPtr {
start: t.0,
pages: t.1,
len: t.2,
gen: t.3,
}
}
pub(super) fn apply_delta_chain(
pager: &mut Pager,
mut catalog: Vec<CatalogEntry>,
head: RunPtr,
) -> Result<(Vec<CatalogEntry>, Vec<RunPtr>, usize)> {
let mut deltas: Vec<(RunPtr, Vec<(u32, CatalogEntry)>)> = Vec::new();
let mut at = head;
while !at.is_empty() {
if deltas.len() > 4096 {
return Err(MnemoError::Invalid("catalog delta chain too long".into()));
}
let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
let d: CatalogDelta =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
deltas.push((at, d.rows));
at = tuple_ptr(d.prev);
}
deltas.reverse();
let mut rows = 0usize;
let mut chain = Vec::with_capacity(deltas.len());
for (ptr, d) in deltas {
rows += d.len();
chain.push(ptr);
for (i, row) in d {
let i = i as usize;
match i.cmp(&catalog.len()) {
std::cmp::Ordering::Less => catalog[i] = row,
std::cmp::Ordering::Equal => catalog.push(row),
std::cmp::Ordering::Greater => {
return Err(MnemoError::Invalid("catalog delta skips a row".into()))
}
}
}
}
Ok((catalog, chain, rows))
}
#[derive(Serialize)]
pub(super) struct IndexDeltaRef<'a> {
pub(super) prev: (u64, u64, u64, u64),
pub(super) ops: &'a [IndexOp],
}
#[derive(Deserialize)]
pub(super) struct IndexDeltaOwned {
pub(super) prev: (u64, u64, u64, u64),
pub(super) ops: Vec<IndexOp>,
}
pub(super) fn apply_index_chain(
pager: &mut Pager,
ann: &mut IvfPqIndex,
head: RunPtr,
) -> Result<(Vec<RunPtr>, usize)> {
let mut deltas: Vec<(RunPtr, Vec<IndexOp>)> = Vec::new();
let mut at = head;
while !at.is_empty() {
if deltas.len() > 4096 {
return Err(MnemoError::Invalid("index delta chain too long".into()));
}
let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
let d: IndexDeltaOwned =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
deltas.push((at, d.ops));
at = tuple_ptr(d.prev);
}
deltas.reverse();
let mut chain = Vec::with_capacity(deltas.len());
let mut n = 0usize;
for (ptr, ops) in deltas {
n += ops.len();
chain.push(ptr);
ann.apply_ops(ops)?;
}
Ok((chain, n))
}
pub(super) fn index_chain_runs(pager: &mut Pager, head: RunPtr) -> Vec<RunPtr> {
let mut runs = Vec::new();
let mut at = head;
while !at.is_empty() && runs.len() <= 4096 {
let Ok(buf) = read_run_bytes(pager, at.start, at.pages, at.len, at.gen) else {
break;
};
let Ok(d) = rmp_serde::from_slice::<IndexDeltaOwned>(&buf) else {
break;
};
runs.push(at);
at = tuple_ptr(d.prev);
}
runs.reverse();
runs
}
pub(super) fn load_catalog_v4(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
) -> Result<Vec<CatalogEntryV4>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, 0)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
pub(super) fn migrate_v4_catalog(
pager: &mut Pager,
v4: Vec<CatalogEntryV4>,
) -> Result<Vec<CatalogEntry>> {
let mut out = Vec::with_capacity(v4.len());
for e in &v4 {
let (accessed_at, access_count) = if e.deleted {
(0i64, 0u32)
} else {
let mut buf = Vec::with_capacity(e.len as usize);
for i in 0..e.page_count as u64 {
buf.extend_from_slice(&pager.read_page(e.start_page + i, 0)?);
}
buf.truncate(e.len as usize);
let mem: Memory = rmp_serde::from_slice(&buf)
.map_err(|err| MnemoError::Serialize(err.to_string()))?;
(mem.accessed_at, mem.access_count)
};
out.push(CatalogEntry {
id: e.id,
start_page: e.start_page,
page_count: e.page_count,
len: e.len,
deleted: e.deleted,
accessed_at,
access_count,
gen: 0,
v9: None,
});
}
Ok(out)
}
pub(super) fn load_manifest(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<Snapshot>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, gen)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
pub(super) fn load_cache_directory(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<CacheDirectoryEntry>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, gen)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
pub(super) fn load_index(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
dims: usize,
) -> Result<Option<IvfPqIndex>> {
if pages == 0 {
return Ok(None);
}
if pager.version() < VERSION_V9 {
let buf = read_run_bytes(pager, start, pages, len, gen)?;
let idx: IvfPqIndex =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if idx.dims() != dims {
return Err(MnemoError::Invalid(
"ANN index dimensionality does not match the database".into(),
));
}
return Ok(Some(idx));
}
let err = std::cell::RefCell::new(None);
let mut rd = RunReader::new(pager, &err, start, pages, len, gen);
let mut layout = [0u8; 1];
if rd.read_exact(&mut layout).is_err() {
drop(rd);
return match err.into_inner() {
Some(page_err) => Err(page_err),
None => Ok(None),
};
}
let res: std::result::Result<IvfPqIndex, MnemoError> = match layout[0] {
V9_INDEX_LAYOUT => IvfPqIndex::read_layout2(&mut rd, len.saturating_sub(1)),
V9_INDEX_LAYOUT_MSGPACK => {
rmp_serde::from_read(rd).map_err(|e| MnemoError::Serialize(e.to_string()))
}
_ => return Ok(None),
};
let idx = match (res, err.into_inner()) {
(Ok(i), _) => i,
(Err(_), Some(page_err)) => return Err(page_err),
(Err(e), None) => return Err(e),
};
if idx.dims() != dims {
return Err(MnemoError::Invalid(
"ANN index dimensionality does not match the database".into(),
));
}
Ok(Some(idx))
}
pub(super) fn load_free_list(pager: &mut Pager, ptr: RunPtr) -> Result<FreeList> {
if ptr.is_empty() {
return Ok(FreeList::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let runs: Vec<crate::freelist::FreeRun> =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
FreeList::from_runs(runs)
}
pub(super) fn load_key_slots(pager: &mut Pager, ptr: RunPtr) -> Result<KeySlotRun> {
if ptr.is_empty() {
return Ok(KeySlotRun::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let run: KeySlotRun =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
run.validate()?;
Ok(run)
}