use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use fs4::fs_std::FileExt;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use ulid::Ulid;
use crate::crypto::{self, KdfParams};
use crate::embed::{self, Embedder};
use crate::error::{MnemoError, Result};
use crate::format::{
Header, RunPtr, FLAG_ENCRYPTED, PAGE_SIZE, PAYLOAD, VERSION, VERSION_V9, WRAPPED_DEK_LEN,
};
use crate::freelist::FreeList;
use crate::index::{IndexConfig, IndexInfo, IndexOp, IvfPqIndex};
use crate::keyslot::{KeySlot as KeySlotEntry, KeySlotRun};
use crate::memory::{
self, status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind,
EssenceScope, EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, Memory,
MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_TRANSITIONS,
MNEMO_META_CONTINUES, MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND,
MNEMO_META_KIND_ESSENCE, MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH,
MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
};
use crate::pager::Pager;
use crate::result_cache::{
self, BatchState, CacheBudget, CacheDirectoryEntry, CacheEntry, CacheFlushPolicy, CachePutOpts,
CacheStats, CachedValue, CounterState, DirectoryIndex, SemanticCachePutOpts,
};
use crate::text;
use crate::veccol;
use crate::wal;
const DEFAULT_WAL_PAGES: u64 = 8;
const MIN_WAL_PAGES: u64 = 2;
pub const DEFAULT_MAX_SNAPSHOTS: usize = 256;
pub const INDEX_PERSIST_EVERY_COMMITS: u32 = 64;
pub const INDEX_PERSIST_EVERY_CHANGES: usize = 1_000;
pub const DEFAULT_KEYWORD_WEIGHT: f32 = 0.5;
pub const DEFAULT_VECTOR_CACHE_BUDGET: usize = 96 * 1024 * 1024;
pub const V9_ANN_MIN_FACTS: usize = 20_000;
pub const V9_RERANK_PER_HIT: usize = 50;
const V9_WAL_RUN_MAX_PAGES: u64 = 2;
pub const V9_DEFAULT_MAX_SNAPSHOTS: usize = 16;
const V9_SNAPSHOT_KEEP_SECS: i64 = 24 * 3600;
const V9_PIN_FLOOR_PAGES: u64 = 128;
const V9_INDEX_LAYOUT: u8 = 1;
const VACUUM_PAGES_PER_COMMIT: usize = 4;
const VACUUM_MAX_ROWS: usize = 64;
const FLAG_FLOAT_VECTORS: u32 = 2;
#[derive(Debug, Default, Clone, Copy)]
struct IndexPersist {
forced: bool,
commits: u32,
changes: usize,
}
#[derive(Debug, Default)]
struct LayerCache {
essence: std::collections::HashSet<u128>,
onboarding: std::collections::HashSet<u128>,
threads: std::collections::HashMap<String, ThreadCache>,
links: std::collections::HashMap<String, (String, u128)>,
cold_turns: std::collections::HashMap<u32, Vec<u128>>,
}
#[derive(Debug, Default)]
struct ThreadCache {
records: std::collections::HashSet<u128>,
turns: std::collections::HashMap<String, u32>,
max_seq: Option<u64>,
}
fn meta_str<'a>(m: &'a Memory, key: &str) -> Option<&'a str> {
m.metadata.get(key).and_then(|v| v.as_str())
}
fn is_onboarding(m: &Memory) -> bool {
meta_str(m, "area").is_some_and(|s| s.eq_ignore_ascii_case("onboarding"))
}
impl LayerCache {
fn add(&mut self, m: &Memory) {
let id = m.id.0;
let kind = meta_str(m, MNEMO_META_KIND);
if kind == Some(MNEMO_META_KIND_ESSENCE) {
self.essence.insert(id);
}
if is_onboarding(m) {
self.onboarding.insert(id);
}
let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
return;
};
let tc = self.threads.entry(thread.to_string()).or_default();
tc.records.insert(id);
if kind == Some(MNEMO_META_KIND_TURN) {
if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
*tc.turns.entry(turn_id.to_string()).or_insert(0) += 1;
}
if let Some(seq) = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64()) {
tc.max_seq = Some(tc.max_seq.map_or(seq, |s| s.max(seq)));
}
}
if kind == Some(MNEMO_META_KIND_THREAD_LINK) {
if let Some(from) = meta_str(m, MNEMO_META_CONTINUES) {
self.links
.entry(thread.to_string())
.or_insert_with(|| (from.to_string(), id));
}
}
}
fn remove(&mut self, m: &Memory) {
let id = m.id.0;
self.essence.remove(&id);
self.onboarding.remove(&id);
let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
return;
};
if self.links.get(thread).is_some_and(|(_, link)| *link == id) {
self.links.remove(thread);
}
let Some(tc) = self.threads.get_mut(thread) else {
return;
};
tc.records.remove(&id);
if meta_str(m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN) {
if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
if let Some(n) = tc.turns.get_mut(turn_id) {
*n -= 1;
if *n == 0 {
tc.turns.remove(turn_id);
}
}
}
}
if tc.records.is_empty() {
self.threads.remove(thread);
}
}
}
fn filter_thread_pin(f: &Filter) -> Option<&str> {
f.terms.iter().find_map(|t| match t {
FilterTerm::Eq(k, Value::String(s)) if k == MNEMO_META_THREAD => Some(s.as_str()),
_ => None,
})
}
fn filter_bypasses_ann(f: &Filter) -> bool {
let non_ann = |v: &Value| {
matches!(
v.as_str(),
Some(MNEMO_META_KIND_TURN)
| Some(MNEMO_META_KIND_ESSENCE)
| Some(MNEMO_META_KIND_THREAD_LINK)
)
};
filter_thread_pin(f).is_some()
|| f.terms.iter().any(|t| match t {
FilterTerm::Eq(k, v) => k == MNEMO_META_KIND && non_ann(v),
FilterTerm::In(k, vs) => {
k == MNEMO_META_KIND && !vs.is_empty() && vs.iter().all(non_ann)
}
_ => false,
})
}
const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20;
fn scan_header_recovery_candidates(
file: &mut File,
secret: &[u8],
) -> Result<Option<Vec<wal::Frame>>> {
let file_len = file.seek(SeekFrom::End(0))?;
if file_len < PAGE_SIZE as u64 * 2 {
return Ok(None); }
let mut best: Option<(u64, Vec<wal::Frame>)> = None;
let page_sz = PAGE_SIZE as u64;
let mut offset = page_sz;
let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
while offset < file_len {
let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
chunk.resize(want, 0);
file.seek(SeekFrom::Start(offset))?;
let mut filled = 0;
while filled < want {
match file.read(&mut chunk[filled..])? {
0 => break,
n => filled += n,
}
}
if filled == 0 {
break;
}
let mut inner = 0usize;
while inner + 4 <= filled {
if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
let page_no = (offset + inner as u64) / page_sz;
if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
let keep = match &best {
None => true,
Some((id, _)) => txn_id > *id,
};
if keep {
best = Some((txn_id, frames));
}
}
}
inner += PAGE_SIZE;
}
offset += filled as u64;
}
Ok(best.map(|(_, f)| f))
}
fn try_recover_candidate(
file: &mut File,
candidate_page_no: u64,
secret: &[u8],
) -> Result<Option<(u64, Vec<wal::Frame>)>> {
let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
return Ok(None);
};
let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
Some((_, b)) => b,
None => return Ok(None),
};
if header_bytes.len() != PAGE_SIZE {
return Ok(None);
}
let mut hbuf = [0u8; PAGE_SIZE];
hbuf.copy_from_slice(header_bytes);
let candidate = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(_) => return Ok(None),
};
if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
return Ok(None);
}
let kdf = KdfParams {
m_cost: candidate.m_cost,
t_cost: candidate.t_cost,
p_cost: candidate.p_cost,
};
let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
Ok(k) => k,
Err(_) => return Ok(None),
};
let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
Ok(d) => d,
Err(_) => return Ok(None),
};
if candidate.validate_seal(&dek).is_err() {
return Ok(None);
}
Ok(Some((txn_id, frames)))
}
#[derive(Clone, Copy, Debug)]
pub struct MnemoConfig {
pub dimensions: usize,
pub kdf: KdfParams,
pub wal_pages_initial: u64,
pub max_snapshots: usize,
#[doc(hidden)]
pub format_version: u16,
pub store_float_vectors: bool,
}
impl Default for MnemoConfig {
fn default() -> Self {
Self {
dimensions: 768,
kdf: KdfParams::secure(),
wal_pages_initial: DEFAULT_WAL_PAGES,
max_snapshots: DEFAULT_MAX_SNAPSHOTS,
format_version: 0,
store_float_vectors: false,
}
}
}
fn resolve_format_version(requested: u16) -> Result<u16> {
match requested {
0 => Ok(crate::format::DEFAULT_NEW_VERSION),
VERSION | VERSION_V9 => Ok(requested),
v => Err(MnemoError::Invalid(format!(
"format_version {v} is not supported for new files (use 8 or 9)"
))),
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct CatalogEntry {
id: u128,
start_page: u64,
page_count: u32,
len: u32,
deleted: bool,
accessed_at: i64,
access_count: u32,
#[serde(default, skip_serializing_if = "crate::format::is_zero_u64")]
gen: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
v9: Option<RowV9>,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
struct RowV9 {
vec_page: u64,
vec_slot: u16,
vec_gen: u64,
rec_slot: u16,
kind: u8,
flags: u8,
memory_type: u8,
thread_hash: u32,
key_hash: u32,
agent_hash: u32,
created_at: i64,
importance: f32,
expires_at: i64,
doc: u32,
doc_len: u16,
}
const REC_SPAN: u16 = u16::MAX;
const PAGE_KIND_RECORD: u8 = 1;
const REC_PAGE_CAP: usize = PAYLOAD - veccol::PAGE_HEADER;
const REC_SLOT_BYTES: usize = 4;
const V9_TURN_CHUNK_BYTES: usize = 7_900;
const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD];
const TURN_ZSTD_LEVEL: i32 = 3;
const TURN_ZSTD_MIN_BYTES: usize = 256;
const RECORD_PLAIN_MAX: u64 = 64 << 20;
pub const DEFAULT_TURN_COMPRESS_AFTER_SECS: i64 = 7 * 24 * 3600;
const COLD_TURNS_PER_COMMIT: usize = 256;
fn turn_zstd_enabled() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("MNEMO_DEV_TURN_ZSTD").as_deref() != Ok("0"))
}
fn compress_turn_record(bytes: Vec<u8>) -> Vec<u8> {
if bytes.len() < TURN_ZSTD_MIN_BYTES || !turn_zstd_enabled() {
return bytes;
}
let c = ZSTD_COMPRESSOR.with(|cell| {
let mut slot = cell.borrow_mut();
if slot.is_none() {
*slot = zstd::bulk::Compressor::new(TURN_ZSTD_LEVEL).ok();
}
let out = slot.as_mut().and_then(|z| z.compress(&bytes).ok());
if out.is_none() {
*slot = None;
}
out
});
match c {
Some(c) if c.len() * 8 <= bytes.len() * 7 => c,
_ => bytes,
}
}
thread_local! {
static ZSTD_COMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Compressor<'static>>> =
const { std::cell::RefCell::new(None) };
static ZSTD_DECOMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Decompressor<'static>>> =
const { std::cell::RefCell::new(None) };
}
pub(crate) fn plain_record(bytes: &[u8]) -> Result<std::borrow::Cow<'_, [u8]>> {
if !bytes.starts_with(&ZSTD_MAGIC) {
return Ok(std::borrow::Cow::Borrowed(bytes));
}
let bad = |why: &str| MnemoError::Invalid(format!("compressed record: {why}"));
let size = match zstd::zstd_safe::get_frame_content_size(bytes) {
Ok(Some(n)) if n <= RECORD_PLAIN_MAX => n as usize,
Ok(Some(_)) => return Err(bad("too large")),
_ => return Err(bad("no content size")),
};
let out = ZSTD_DECOMPRESSOR.with(|cell| {
let mut slot = cell.borrow_mut();
if slot.is_none() {
*slot = Some(zstd::bulk::Decompressor::new().map_err(|e| bad(&e.to_string()))?);
}
let out = match slot.as_mut() {
Some(z) => z.decompress(bytes, size).map_err(|e| bad(&e.to_string())),
None => Err(bad("no context")),
};
if out.is_err() {
*slot = None;
}
out
})?;
if out.len() != size {
return Err(bad("length mismatch"));
}
Ok(std::borrow::Cow::Owned(out))
}
fn vacuum_pick(sparse: Vec<(u64, usize)>) -> Vec<u64> {
let mut rows = 0usize;
let mut out = Vec::new();
for (p, n) in sparse {
if out.len() == VACUUM_PAGES_PER_COMMIT {
break;
}
if rows + n > VACUUM_MAX_ROWS {
continue;
}
rows += n;
out.push(p);
}
out
}
fn encode_record_page(gen: u64, recs: &[&[u8]]) -> Vec<u8> {
let dir = recs.len() * REC_SLOT_BYTES;
let data: usize = recs.iter().map(|r| r.len()).sum();
let mut out = Vec::with_capacity(veccol::PAGE_HEADER + dir + data);
out.extend_from_slice(&[PAGE_KIND_RECORD, 0, 0, 0]);
out.extend_from_slice(&(recs.len() as u16).to_le_bytes());
out.extend_from_slice(&((dir + data) as u16).to_le_bytes());
out.extend_from_slice(&gen.to_le_bytes());
let mut off = veccol::PAGE_HEADER + dir;
for r in recs {
out.extend_from_slice(&(off as u16).to_le_bytes());
out.extend_from_slice(&(r.len() as u16).to_le_bytes());
off += r.len();
}
for r in recs {
out.extend_from_slice(r);
}
out
}
fn record_slot(page: &[u8], slot: u16) -> Result<Vec<u8>> {
let bad = || MnemoError::Invalid("malformed record page".into());
if page.len() < veccol::PAGE_HEADER || page[0] != PAGE_KIND_RECORD {
return Err(bad());
}
let n = u16::from_le_bytes([page[4], page[5]]);
if slot >= n {
return Err(bad());
}
let d = veccol::PAGE_HEADER + slot as usize * REC_SLOT_BYTES;
let dir = page.get(d..d + REC_SLOT_BYTES).ok_or_else(bad)?;
let off = u16::from_le_bytes([dir[0], dir[1]]) as usize;
let len = u16::from_le_bytes([dir[2], dir[3]]) as usize;
page.get(off..off + len).map(|b| b.to_vec()).ok_or_else(bad)
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct Bm25Manifest {
tokenizer: u8,
next_doc: u32,
segments: Vec<(u64, u64, u64, u64)>,
#[serde(default)]
seg_docs: Vec<u32>,
}
impl Default for Bm25Manifest {
fn default() -> Self {
Bm25Manifest {
tokenizer: text::TOKENIZER_V1,
next_doc: 1,
segments: Vec::new(),
seg_docs: Vec::new(),
}
}
}
#[derive(Debug, Default)]
struct DocStats {
docs: std::collections::HashMap<u32, (usize, u16)>,
avgdl: f32,
total: u64,
}
impl DocStats {
fn remove(&mut self, doc: u32) {
if let Some((_, len)) = self.docs.remove(&doc) {
self.total = self.total.saturating_sub(len as u64);
self.refresh();
}
}
fn insert(&mut self, doc: u32, row: usize, len: u16) {
if let Some((_, old)) = self.docs.insert(doc, (row, len)) {
self.total = self.total.saturating_sub(old as u64);
}
self.total += len as u64;
self.refresh();
}
fn refresh(&mut self) {
self.avgdl = if self.docs.is_empty() {
1.0
} else {
self.total as f32 / self.docs.len() as f32
};
}
}
#[derive(Debug, Default)]
struct Bm25State {
manifest: Bm25Manifest,
dicts: Option<Vec<Vec<text::DictEntry>>>,
pending: Vec<(u32, Vec<(String, u8)>)>,
stats: Option<DocStats>,
merged_docs: u32,
}
type Bm25Segments = (Vec<(u64, u64, u64, u64)>, Vec<u32>);
const BM25_MAX_SEGMENTS: usize = 8;
fn bm25_kind(kind: u8) -> bool {
kind == ROW_KIND_FACT || kind == ROW_KIND_ONBOARDING
}
fn read_run_range(pager: &mut Pager, ptr: RunPtr, off: u64, len: u64) -> Result<Vec<u8>> {
let mut out = Vec::with_capacity(len as usize);
if len == 0 {
return Ok(out);
}
if off + len > ptr.len {
return Err(MnemoError::Invalid("BM25 range past its run".into()));
}
let p = PAYLOAD as u64;
let (first, last) = (off / p, (off + len - 1) / p);
for i in first..=last {
let page = pager.read_page(ptr.start + i, ptr.gen)?;
let lo = if i == first { (off % p) as usize } else { 0 };
let hi = if i == last {
((off + len - 1) % p) as usize + 1
} else {
PAYLOAD
};
out.extend_from_slice(&page[lo..hi]);
}
Ok(out)
}
fn load_segment_dict(pager: &mut Pager, ptr: RunPtr) -> Result<Vec<text::DictEntry>> {
let head = read_run_range(pager, ptr, 0, 4)?;
let n = u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
let bytes = read_run_range(pager, ptr, 4, n)?;
rmp_serde::from_slice(&bytes).map_err(|e| MnemoError::Serialize(e.to_string()))
}
fn segment_postings(
pager: &mut Pager,
ptr: RunPtr,
dict: &[text::DictEntry],
term: &str,
) -> Result<Vec<(u32, u8)>> {
let Ok(i) = dict.binary_search_by(|e| e.term.as_str().cmp(term)) else {
return Ok(Vec::new());
};
let head = read_run_range(pager, ptr, 0, 4)?;
let base = 4 + u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
let e = &dict[i];
let bytes = read_run_range(pager, ptr, base + e.off as u64, e.len as u64)?;
text::decode_postings(&bytes)
}
fn load_bm25_manifest(pager: &mut Pager, ptr: RunPtr) -> Result<Bm25Manifest> {
if ptr.is_empty() {
return Ok(Bm25Manifest::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let m: Bm25Manifest =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if m.tokenizer != text::TOKENIZER_V1 {
return Err(MnemoError::Invalid(format!(
"BM25 tokenizer {} is newer than this build understands",
m.tokenizer
)));
}
Ok(m)
}
#[derive(Debug, Default)]
struct RecPages {
hot: std::collections::HashMap<usize, Vec<u8>>,
pending: Vec<usize>,
page_rows: std::collections::HashMap<u64, Vec<usize>>,
}
impl RecPages {
fn from_catalog(catalog: &[CatalogEntry]) -> RecPages {
let mut recs = RecPages::default();
for (i, e) in catalog.iter().enumerate() {
if e.deleted || e.start_page == 0 {
continue;
}
if let Some(r) = e.v9 {
if r.rec_slot != REC_SPAN {
recs.page_rows.entry(e.start_page).or_default().push(i);
}
}
}
recs
}
}
const ROW_SUPERSEDED: u8 = 1;
const ROW_HAS_TTL: u8 = 2;
const ROW_SHARED: u8 = 4;
const ROW_SETTLED: u8 = 8;
const ROW_KIND_FACT: u8 = 0;
const ROW_KIND_TURN: u8 = 1;
const ROW_KIND_ESSENCE: u8 = 2;
const ROW_KIND_THREAD_LINK: u8 = 3;
const ROW_KIND_ONBOARDING: u8 = 4;
fn fnv1a32(parts: &[&[u8]]) -> u32 {
let mut h: u32 = 0x811c_9dc5;
for (i, p) in parts.iter().enumerate() {
if i > 0 {
h ^= 0xff;
h = h.wrapping_mul(0x0100_0193);
}
for &b in p.iter() {
h ^= b as u32;
h = h.wrapping_mul(0x0100_0193);
}
}
h.max(1)
}
fn memory_type_code(t: MemoryType) -> u8 {
match t {
MemoryType::Episodic => 0,
MemoryType::Semantic => 1,
MemoryType::Procedural => 2,
MemoryType::Working => 3,
}
}
fn row_tags(m: &Memory) -> RowV9 {
let kind = match meta_str(m, MNEMO_META_KIND) {
Some(MNEMO_META_KIND_TURN) => ROW_KIND_TURN,
Some(MNEMO_META_KIND_ESSENCE) => ROW_KIND_ESSENCE,
Some(MNEMO_META_KIND_THREAD_LINK) => ROW_KIND_THREAD_LINK,
_ if is_onboarding(m) => ROW_KIND_ONBOARDING,
_ => ROW_KIND_FACT,
};
let mut flags = 0u8;
if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
flags |= ROW_SUPERSEDED;
}
if m.ttl_secs.is_some() {
flags |= ROW_HAS_TTL;
}
if m.scope == Scope::Shared {
flags |= ROW_SHARED;
}
let key_hash = meta_str(m, MNEMO_META_KEY).map_or(0, |k| {
let slot = Mnemo::key_slot_for(m, k);
fnv1a32(&[
slot.scope_owner.as_deref().unwrap_or("").as_bytes(),
slot.key.as_bytes(),
])
});
RowV9 {
kind,
flags,
memory_type: memory_type_code(m.memory_type),
thread_hash: meta_str(m, MNEMO_META_THREAD).map_or(0, |t| fnv1a32(&[t.as_bytes()])),
key_hash,
agent_hash: fnv1a32(&[m.agent_id.as_bytes()]),
created_at: m.created_at,
importance: m.importance,
expires_at: m.ttl_secs.map_or(0, |t| m.created_at.saturating_add(t)),
..RowV9::default()
}
}
#[derive(Debug, Default)]
struct VecColumn {
data: Vec<u16>,
norms: Vec<f32>,
have: Vec<bool>,
resident: bool,
hot: std::collections::HashMap<usize, (Vec<u16>, f32)>,
pending: Vec<usize>,
page_rows: std::collections::HashMap<u64, Vec<usize>>,
}
impl VecColumn {
fn from_catalog(catalog: &[CatalogEntry]) -> VecColumn {
let mut col = VecColumn::default();
for (i, e) in catalog.iter().enumerate() {
if e.deleted {
continue;
}
if let Some(r) = e.v9 {
if r.vec_page != 0 {
col.page_rows.entry(r.vec_page).or_default().push(i);
}
}
}
col
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct CatalogEntryV4 {
id: u128,
start_page: u64,
page_count: u32,
len: u32,
deleted: bool,
}
struct FlushPrelude {
cat_bytes: Option<Vec<u8>>,
cat_delta: bool,
idx_delta: bool,
bm25_seg: Option<(Vec<u8>, Option<usize>)>,
idx_bytes: Option<Vec<u8>>,
cache_bytes: Option<Vec<u8>>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct Snapshot {
txn_id: u64,
created_at: i64,
catalog_start: u64,
catalog_pages: u64,
catalog_len: u64,
index_start: u64,
index_pages: u64,
index_len: u64,
memory_count: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
gens: Option<SnapshotGens>,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
struct SnapshotGens {
catalog: u64,
index: u64,
#[serde(default)]
delta: (u64, u64, u64, u64),
#[serde(default)]
index_delta: (u64, u64, u64, u64),
#[serde(default)]
bm25: (u64, u64, u64, u64),
}
impl Snapshot {
fn catalog_gen(&self) -> u64 {
self.gens.map_or(0, |g| g.catalog)
}
fn index_gen(&self) -> u64 {
self.gens.map_or(0, |g| g.index)
}
fn delta_head(&self) -> RunPtr {
self.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.delta))
}
fn index_delta_head(&self) -> RunPtr {
self.gens
.map_or(RunPtr::default(), |g| tuple_ptr(g.index_delta))
}
}
#[derive(Clone, Copy, Debug)]
pub struct SnapshotInfo {
pub txn_id: u64,
pub created_at: i64,
pub memory_count: u64,
}
#[derive(Clone, Debug)]
pub struct RecallRequest {
pub query: Vec<f32>,
pub top_k: usize,
pub memory_types: Option<Vec<MemoryType>>,
pub agent_id: Option<String>,
pub metric: Metric,
pub weights: ScoreWeights,
pub n_probe: Option<usize>,
pub n_rerank: Option<usize>,
pub track_access: bool,
pub filter: Filter,
pub max_bytes: Option<usize>,
pub keywords: Option<String>,
pub keyword_weight: f32,
pub bm25_k1: f32,
pub bm25_b: f32,
}
impl RecallRequest {
pub fn new(query: Vec<f32>) -> Self {
Self {
query,
top_k: 10,
memory_types: None,
agent_id: None,
metric: Metric::Cosine,
weights: ScoreWeights::default(),
n_probe: None,
n_rerank: None,
track_access: true,
filter: Filter::default(),
max_bytes: None,
keywords: None,
keyword_weight: DEFAULT_KEYWORD_WEIGHT,
bm25_k1: 1.2,
bm25_b: 0.75,
}
}
pub fn keywords_only(keywords: impl Into<String>) -> Self {
Self::new(Vec::new()).keywords(keywords)
}
pub fn keywords(mut self, text: impl Into<String>) -> Self {
self.keywords = Some(text.into());
self
}
pub fn keyword_weight(mut self, w: f32) -> Self {
self.keyword_weight = w;
self
}
pub fn filter(mut self, filter: Filter) -> Self {
self.filter = filter;
self
}
pub fn max_bytes(mut self, n: usize) -> Self {
self.max_bytes = Some(n);
self
}
pub fn from_text(embedder: &dyn Embedder, query_text: &str) -> Result<Self> {
let mut vectors = embedder.embed(&[query_text])?;
let v = vectors.pop().ok_or_else(|| {
MnemoError::Invalid("embedder returned no vector for the query text".into())
})?;
Ok(Self::new(v))
}
pub fn top_k(mut self, k: usize) -> Self {
self.top_k = k;
self
}
pub fn types(mut self, t: Vec<MemoryType>) -> Self {
self.memory_types = Some(t);
self
}
pub fn agent(mut self, agent_id: impl Into<String>) -> Self {
self.agent_id = Some(agent_id.into());
self
}
pub fn metric(mut self, metric: Metric) -> Self {
self.metric = metric;
self
}
pub fn weights(mut self, weights: ScoreWeights) -> Self {
self.weights = weights;
self
}
pub fn n_probe(mut self, n: usize) -> Self {
self.n_probe = Some(n);
self
}
pub fn n_rerank(mut self, n: usize) -> Self {
self.n_rerank = Some(n);
self
}
pub fn track_access(mut self, track: bool) -> Self {
self.track_access = track;
self
}
}
#[derive(Clone, Debug)]
pub struct RecallResult {
pub memory: Memory,
pub score: f32,
pub similarity: f32,
}
#[derive(Clone, Debug)]
pub struct RecallResponse {
pub hits: Vec<RecallResult>,
pub truncated: bool,
pub used_exact_scan: bool,
}
#[doc(hidden)]
#[derive(Clone, Debug, Default, serde::Serialize)]
pub struct BudgetReport {
pub file_bytes: u64,
pub page_size: u64,
pub header_bytes: u64,
pub record_page_bytes: u64,
pub record_payload_bytes: u64,
pub deleted_record_page_bytes: u64,
pub catalog_run_bytes: u64,
pub catalog_payload_bytes: u64,
pub index_run_bytes: u64,
pub index_payload_bytes: u64,
pub index_start_page: u64,
pub wal_bytes: u64,
pub snapshot_manifest_bytes: u64,
pub cache_dir_run_bytes: u64,
pub unreferenced_bytes: u64,
pub live_records: u64,
pub ram_catalog: u64,
pub ram_id_map: u64,
pub ram_ann_centroids: u64,
pub ram_ann_codebook: u64,
pub ram_ann_posting_ids: u64,
pub ram_ann_posting_codes: u64,
pub ram_ann_assignment: u64,
pub ram_cache_dir: u64,
pub ram_key_caches: u64,
pub ram_vector_column: u64,
pub ram_total: u64,
}
#[derive(Clone, Debug)]
pub struct Stats {
pub memories: usize,
pub deleted: usize,
pub dimensions: usize,
pub file_bytes: u64,
pub agents: Vec<String>,
pub encrypted: bool,
pub created_at: i64,
pub index: Option<IndexInfo>,
pub wal_pages: u64,
pub fsyncs_last_flush: u32,
pub format_version: u16,
}
#[derive(Clone, Copy, Debug, Default)]
pub struct CompactReport {
pub before: usize,
pub after: usize,
pub cache_before: usize,
pub cache_after: usize,
}
pub struct Mnemo {
pager: Pager,
header: Header,
catalog: Vec<CatalogEntry>,
index: IdIndex,
#[allow(dead_code)]
path: PathBuf,
dimensions: usize,
kdf: KdfParams,
dirty_catalog: bool,
ann: Option<IvfPqIndex>,
dirty_index: bool,
index_persist: IndexPersist,
ann_unloaded: bool,
manifest: Vec<Snapshot>,
max_snapshots: usize,
read_only: bool,
cache_dir: Vec<CacheDirectoryEntry>,
cache_idx: DirectoryIndex,
dirty_cache: bool,
cache_budgets: std::collections::HashMap<String, CacheBudget>,
cache_flush_policy: CacheFlushPolicy,
cache_batch: BatchState,
cache_counters: CounterState,
embedder: Option<std::sync::Arc<dyn Embedder>>,
fsyncs_last_flush: u32,
agent_set: Option<std::collections::BTreeSet<String>>,
key_index: Option<std::collections::HashMap<KeySlot, u128>>,
superseded: Option<std::collections::HashSet<u128>>,
layers: Option<LayerCache>,
turn_compress_after: i64,
free: FreeList,
pending_free: Vec<(u64, u64)>,
free_dirty: bool,
key_slots: KeySlotRun,
dirty_keys: bool,
gen_leased: bool,
col: VecColumn,
recs: RecPages,
bm25: Bm25State,
index_layout_override: Option<u8>,
kw: Option<std::collections::HashMap<u128, f32>>,
dirty_rows: std::collections::BTreeSet<usize>,
delta_chain: Vec<RunPtr>,
delta_rows: usize,
index_chain: Vec<RunPtr>,
index_chain_ops: usize,
index_rebase: bool,
index_delta_ops: Option<Vec<IndexOp>>,
catalog_rewrite: bool,
snapshot_cap_explicit: bool,
direct_writes: bool,
vector_cache_budget: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) struct KeySlot {
pub(crate) scope_owner: Option<String>,
pub(crate) key: String,
}
fn open_with_lock(path: &Path, write: bool, create: bool) -> Result<File> {
let mut opts = OpenOptions::new();
opts.read(true);
if write {
opts.write(true);
}
if create {
opts.create(true).truncate(true);
}
let file = opts.open(path)?;
let result = if write {
FileExt::try_lock_exclusive(&file)
} else {
FileExt::try_lock_shared(&file)
};
match result {
Ok(()) => Ok(file),
Err(e) if is_lock_conflict(&e) => Err(MnemoError::Locked {
path: path.to_path_buf(),
}),
Err(e) => Err(MnemoError::Io(e)),
}
}
fn paths_refer_to_same_file(a: &Path, b: &Path) -> Option<bool> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let ma = std::fs::metadata(a).ok()?;
let mb = std::fs::metadata(b).ok()?;
Some(ma.dev() == mb.dev() && ma.ino() == mb.ino())
}
#[cfg(not(unix))]
{
let ca = std::fs::canonicalize(a).ok()?;
let cb = std::fs::canonicalize(b).ok()?;
Some(ca == cb)
}
}
fn temp_sibling(dest: &Path) -> PathBuf {
let mut buf = dest.as_os_str().to_os_string();
buf.push(".export-tmp");
PathBuf::from(buf)
}
fn is_lock_conflict(e: &std::io::Error) -> bool {
if e.kind() == std::io::ErrorKind::WouldBlock {
return true;
}
#[cfg(windows)]
if let Some(code) = e.raw_os_error() {
if code == 33 || code == 32 {
return true;
}
}
false
}
fn read_run_bytes(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<u8>> {
let mut buf = Vec::with_capacity(len as usize);
for i in 0..pages {
buf.extend_from_slice(&pager.read_page(start + i, gen)?);
}
buf.truncate(len as usize);
Ok(buf)
}
struct RunReader<'a> {
pager: &'a mut Pager,
err: &'a std::cell::RefCell<Option<MnemoError>>,
next_page: u64,
end_page: u64,
gen: u64,
remaining: u64,
buf: Vec<u8>,
pos: usize,
}
impl<'a> RunReader<'a> {
fn new(
pager: &'a mut Pager,
err: &'a std::cell::RefCell<Option<MnemoError>>,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Self {
RunReader {
pager,
err,
next_page: start,
end_page: start + pages,
gen,
remaining: len,
buf: Vec::new(),
pos: 0,
}
}
}
impl Read for RunReader<'_> {
#[inline]
fn read_exact(&mut self, out: &mut [u8]) -> std::io::Result<()> {
if self.buf.len() - self.pos >= out.len() {
out.copy_from_slice(&self.buf[self.pos..self.pos + out.len()]);
self.pos += out.len();
return Ok(());
}
let mut done = 0;
while done < out.len() {
match self.read(&mut out[done..])? {
0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
n => done += n,
}
}
Ok(())
}
#[inline]
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
if self.pos == self.buf.len() {
if self.remaining == 0 || self.next_page >= self.end_page {
return Ok(0);
}
let mut page = match self.pager.read_page_uncached(self.next_page, self.gen) {
Ok(p) => p,
Err(e) => {
let msg = e.to_string();
*self.err.borrow_mut() = Some(e);
return Err(std::io::Error::other(msg));
}
};
self.next_page += 1;
page.truncate((self.remaining as usize).min(page.len()));
self.remaining -= page.len() as u64;
self.buf = page;
self.pos = 0;
}
let n = out.len().min(self.buf.len() - self.pos);
out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]);
self.pos += n;
Ok(n)
}
}
struct ExactVec<T> {
cap: usize,
_t: std::marker::PhantomData<T>,
}
impl<'de, T: Deserialize<'de>> serde::de::DeserializeSeed<'de> for ExactVec<T> {
type Value = Vec<T>;
fn deserialize<D: serde::Deserializer<'de>>(
self,
d: D,
) -> std::result::Result<Vec<T>, D::Error> {
d.deserialize_seq(self)
}
}
impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for ExactVec<T> {
type Value = Vec<T>;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a sequence")
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(
self,
mut seq: A,
) -> std::result::Result<Vec<T>, A::Error> {
let mut v = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(self.cap));
while let Some(x) = seq.next_element()? {
v.push(x);
}
Ok(v)
}
}
fn load_catalog(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<CatalogEntry>> {
if pages == 0 {
return Ok(Vec::new());
}
let cap = (len / 8) as usize;
if pager.version() < VERSION_V9 {
let buf = read_run_bytes(pager, start, pages, len, gen)?;
let mut de = rmp_serde::Deserializer::from_read_ref(&buf);
return serde::de::DeserializeSeed::deserialize(
ExactVec::<CatalogEntry> {
cap,
_t: std::marker::PhantomData,
},
&mut de,
)
.map_err(|e| MnemoError::Serialize(e.to_string()));
}
let err = std::cell::RefCell::new(None);
let mut de = rmp_serde::Deserializer::new(RunReader::new(pager, &err, start, pages, len, gen));
let res = serde::de::DeserializeSeed::deserialize(
ExactVec::<CatalogEntry> {
cap,
_t: std::marker::PhantomData,
},
&mut de,
);
drop(de);
match (res, err.into_inner()) {
(Ok(v), _) => Ok(v),
(Err(_), Some(page_err)) => Err(page_err),
(Err(e), None) => Err(MnemoError::Serialize(e.to_string())),
}
}
#[derive(Serialize, Deserialize)]
struct CatalogDelta {
prev: (u64, u64, u64, u64),
rows: Vec<(u32, CatalogEntry)>,
}
fn ptr_tuple(p: RunPtr) -> (u64, u64, u64, u64) {
(p.start, p.pages, p.len, p.gen)
}
fn tuple_ptr(t: (u64, u64, u64, u64)) -> RunPtr {
RunPtr {
start: t.0,
pages: t.1,
len: t.2,
gen: t.3,
}
}
fn apply_delta_chain(
pager: &mut Pager,
mut catalog: Vec<CatalogEntry>,
head: RunPtr,
) -> Result<(Vec<CatalogEntry>, Vec<RunPtr>, usize)> {
let mut deltas: Vec<(RunPtr, Vec<(u32, CatalogEntry)>)> = Vec::new();
let mut at = head;
while !at.is_empty() {
if deltas.len() > 4096 {
return Err(MnemoError::Invalid("catalog delta chain too long".into()));
}
let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
let d: CatalogDelta =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
deltas.push((at, d.rows));
at = tuple_ptr(d.prev);
}
deltas.reverse();
let mut rows = 0usize;
let mut chain = Vec::with_capacity(deltas.len());
for (ptr, d) in deltas {
rows += d.len();
chain.push(ptr);
for (i, row) in d {
let i = i as usize;
match i.cmp(&catalog.len()) {
std::cmp::Ordering::Less => catalog[i] = row,
std::cmp::Ordering::Equal => catalog.push(row),
std::cmp::Ordering::Greater => {
return Err(MnemoError::Invalid("catalog delta skips a row".into()))
}
}
}
}
Ok((catalog, chain, rows))
}
#[derive(Serialize)]
struct IndexDeltaRef<'a> {
prev: (u64, u64, u64, u64),
ops: &'a [IndexOp],
}
#[derive(Deserialize)]
struct IndexDeltaOwned {
prev: (u64, u64, u64, u64),
ops: Vec<IndexOp>,
}
fn apply_index_chain(
pager: &mut Pager,
ann: &mut IvfPqIndex,
head: RunPtr,
) -> Result<(Vec<RunPtr>, usize)> {
let mut deltas: Vec<(RunPtr, Vec<IndexOp>)> = Vec::new();
let mut at = head;
while !at.is_empty() {
if deltas.len() > 4096 {
return Err(MnemoError::Invalid("index delta chain too long".into()));
}
let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
let d: IndexDeltaOwned =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
deltas.push((at, d.ops));
at = tuple_ptr(d.prev);
}
deltas.reverse();
let mut chain = Vec::with_capacity(deltas.len());
let mut n = 0usize;
for (ptr, ops) in deltas {
n += ops.len();
chain.push(ptr);
ann.apply_ops(ops)?;
}
Ok((chain, n))
}
fn index_chain_runs(pager: &mut Pager, head: RunPtr) -> Vec<RunPtr> {
let mut runs = Vec::new();
let mut at = head;
while !at.is_empty() && runs.len() <= 4096 {
let Ok(buf) = read_run_bytes(pager, at.start, at.pages, at.len, at.gen) else {
break;
};
let Ok(d) = rmp_serde::from_slice::<IndexDeltaOwned>(&buf) else {
break;
};
runs.push(at);
at = tuple_ptr(d.prev);
}
runs.reverse();
runs
}
fn load_catalog_v4(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
) -> Result<Vec<CatalogEntryV4>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, 0)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
fn migrate_v4_catalog(pager: &mut Pager, v4: Vec<CatalogEntryV4>) -> Result<Vec<CatalogEntry>> {
let mut out = Vec::with_capacity(v4.len());
for e in &v4 {
let (accessed_at, access_count) = if e.deleted {
(0i64, 0u32)
} else {
let mut buf = Vec::with_capacity(e.len as usize);
for i in 0..e.page_count as u64 {
buf.extend_from_slice(&pager.read_page(e.start_page + i, 0)?);
}
buf.truncate(e.len as usize);
let mem: Memory = rmp_serde::from_slice(&buf)
.map_err(|err| MnemoError::Serialize(err.to_string()))?;
(mem.accessed_at, mem.access_count)
};
out.push(CatalogEntry {
id: e.id,
start_page: e.start_page,
page_count: e.page_count,
len: e.len,
deleted: e.deleted,
accessed_at,
access_count,
gen: 0,
v9: None,
});
}
Ok(out)
}
fn load_manifest(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<Snapshot>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, gen)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
fn load_cache_directory(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
) -> Result<Vec<CacheDirectoryEntry>> {
if pages == 0 {
return Ok(Vec::new());
}
let buf = read_run_bytes(pager, start, pages, len, gen)?;
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
}
fn load_index(
pager: &mut Pager,
start: u64,
pages: u64,
len: u64,
gen: u64,
dims: usize,
) -> Result<Option<IvfPqIndex>> {
if pages == 0 {
return Ok(None);
}
if pager.version() < VERSION_V9 {
let buf = read_run_bytes(pager, start, pages, len, gen)?;
let mut idx: IvfPqIndex =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if idx.dims() != dims {
return Err(MnemoError::Invalid(
"ANN index dimensionality does not match the database".into(),
));
}
idx.rebuild_assignment();
return Ok(Some(idx));
}
let err = std::cell::RefCell::new(None);
let mut rd = RunReader::new(pager, &err, start, pages, len, gen);
let mut layout = [0u8; 1];
if rd.read_exact(&mut layout).is_err() {
drop(rd);
return match err.into_inner() {
Some(page_err) => Err(page_err),
None => Ok(None),
};
}
if layout[0] != V9_INDEX_LAYOUT {
return Ok(None);
}
let res: std::result::Result<IvfPqIndex, _> = rmp_serde::from_read(rd);
let mut idx = match (res, err.into_inner()) {
(Ok(i), _) => i,
(Err(_), Some(page_err)) => return Err(page_err),
(Err(e), None) => return Err(MnemoError::Serialize(e.to_string())),
};
if idx.dims() != dims {
return Err(MnemoError::Invalid(
"ANN index dimensionality does not match the database".into(),
));
}
idx.rebuild_assignment();
Ok(Some(idx))
}
fn load_free_list(pager: &mut Pager, ptr: RunPtr) -> Result<FreeList> {
if ptr.is_empty() {
return Ok(FreeList::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let runs: Vec<crate::freelist::FreeRun> =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
FreeList::from_runs(runs)
}
fn load_key_slots(pager: &mut Pager, ptr: RunPtr) -> Result<KeySlotRun> {
if ptr.is_empty() {
return Ok(KeySlotRun::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let run: KeySlotRun =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
run.validate()?;
Ok(run)
}
fn is_manifest(m: &Memory) -> bool {
m.metadata
.get("topic")
.and_then(|v| v.as_str())
.map(|s| s.eq_ignore_ascii_case("manifest"))
.unwrap_or(false)
}
#[derive(Debug, Clone, Default)]
struct IdIndex {
slots: Vec<u32>,
len: usize,
}
const NO_ROW: u32 = u32::MAX;
impl IdIndex {
fn build(catalog: &[CatalogEntry]) -> IdIndex {
let mut t = IdIndex {
slots: vec![NO_ROW; crate::index::table_capacity_for(catalog.len())],
len: 0,
};
for (row, e) in catalog.iter().enumerate() {
t.insert(e.id, row, catalog);
}
t
}
fn get(&self, id: u128, catalog: &[CatalogEntry]) -> Option<usize> {
if self.slots.is_empty() {
return None;
}
let cap = self.slots.len();
let mut i = crate::index::slot_home(id, cap);
loop {
let row = self.slots[i];
if row == NO_ROW {
return None;
}
if catalog[row as usize].id == id {
return Some(row as usize);
}
i = crate::index::next_slot(i, cap);
}
}
fn insert(&mut self, id: u128, row: usize, catalog: &[CatalogEntry]) {
if (self.len + 1) * 4 > self.slots.len() * 3 {
let cap = (self.slots.len() * 2).max(crate::index::table_capacity_for(self.len + 1));
let old = std::mem::replace(&mut self.slots, vec![NO_ROW; cap]);
self.len = 0;
for r in old.into_iter().filter(|&r| r != NO_ROW) {
self.insert(catalog[r as usize].id, r as usize, catalog);
}
}
let cap = self.slots.len();
let mut i = crate::index::slot_home(id, cap);
loop {
let r = self.slots[i];
if r == NO_ROW {
self.slots[i] = row as u32;
self.len += 1;
return;
}
if catalog[r as usize].id == id {
self.slots[i] = row as u32;
return;
}
i = crate::index::next_slot(i, cap);
}
}
fn heap_bytes(&self) -> usize {
self.slots.capacity() * std::mem::size_of::<u32>()
}
}
fn build_id_index(catalog: &[CatalogEntry]) -> IdIndex {
IdIndex::build(catalog)
}
struct PendingTurn {
turn_id: String,
role: String,
created_at: i64,
parts: Vec<String>,
truncation: Option<(String, u64)>,
}
struct IngestPlan {
pending: Vec<PendingTurn>,
next_seq: u64,
report: IngestReport,
}
impl IngestPlan {
fn texts(&self) -> Vec<&str> {
self.pending
.iter()
.flat_map(|p| p.parts.iter().map(String::as_str))
.collect()
}
fn text_count(&self) -> usize {
self.pending.iter().map(|p| p.parts.len()).sum()
}
}
fn written_after_index(e: &CatalogEntry, v9: bool, index_start: u64, index_gen: u64) -> bool {
if v9 {
e.start_page == 0 || e.gen > index_gen
} else {
e.start_page > index_start
}
}
fn is_ann_kind(m: &Memory) -> bool {
!matches!(
m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
Some(MNEMO_META_KIND_TURN)
| Some(MNEMO_META_KIND_ESSENCE)
| Some(MNEMO_META_KIND_THREAD_LINK)
)
}
fn hex_encode(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for b in bytes {
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0x0f) as usize] as char);
}
out
}
fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
if s.len() <= max_bytes {
return s.to_string();
}
let mut cut = 0usize;
for (idx, _) in s.char_indices() {
if idx > max_bytes {
break;
}
cut = idx;
}
s[..cut].to_string()
}
fn status_of(m: &Memory) -> Option<&str> {
m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
}
fn render_essence_item(section: EssenceSection, m: &Memory) -> String {
if section == EssenceSection::Timeline {
let date = format_unix_date(m.created_at);
format!("- [{date}] {}\n", m.content)
} else {
let status = status_of(m).unwrap_or(essence_status::OPEN);
format!("- [{status}] {}\n", m.content)
}
}
fn format_unix_date(secs: i64) -> String {
let days_total = secs.div_euclid(86_400);
let (y, m, d) = civil_from_days(days_total);
format!("{y:04}-{m:02}-{d:02}")
}
fn civil_from_days(z: i64) -> (i64, u32, u32) {
let z = z + 719_468;
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
let doe = (z - era * 146_097) as u64;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe as i64 + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
let y = if m <= 2 { y + 1 } else { y };
(y, m, d)
}
fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
if s.len() <= max_bytes {
return vec![s.to_string()];
}
let mut out = Vec::new();
let mut rest = s;
while rest.len() > max_bytes {
let head = take_utf8_prefix(rest, max_bytes);
let head_bytes = head.len();
out.push(head);
rest = &rest[head_bytes..];
}
if !rest.is_empty() {
out.push(rest.to_string());
}
out
}
impl Mnemo {
pub fn create(
path: impl Into<PathBuf>,
passphrase: &str,
config: MnemoConfig,
) -> Result<Mnemo> {
Self::create_secret(path, passphrase.as_bytes(), config)
}
pub fn create_with_key(
path: impl Into<PathBuf>,
key: &[u8; crypto::KEY_LEN],
config: MnemoConfig,
) -> Result<Mnemo> {
let config = MnemoConfig {
kdf: KdfParams::raw_key(),
..config
};
Self::create_secret(path, key, config)
}
pub fn generate_key() -> [u8; crypto::KEY_LEN] {
*crypto::random_dek()
}
fn create_secret(
path: impl Into<PathBuf>,
secret: &[u8],
config: MnemoConfig,
) -> Result<Mnemo> {
let path: PathBuf = path.into();
if config.dimensions == 0 {
return Err(MnemoError::Invalid("dimensions must be > 0".into()));
}
let version = resolve_format_version(config.format_version)?;
if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
return Err(MnemoError::Invalid(format!(
"a v9 file supports up to {} dimensions for now (one fp16 vector \
must fit a page); vector entries that span pages are not supported yet",
(PAYLOAD - veccol::PAGE_HEADER - 4) / 2
)));
}
let flags = if version >= VERSION_V9 && config.store_float_vectors {
FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
} else {
FLAG_ENCRYPTED
};
let file = open_with_lock(&path, true, true)?;
let salt = crypto::random_salt();
let dek = crypto::random_dek();
let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
let dek_nonce = crypto::random_nonce();
let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
if wrapped.len() != WRAPPED_DEK_LEN {
return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
}
let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
wrapped_dek.copy_from_slice(&wrapped);
let mut header = Header {
version,
page_size: PAGE_SIZE as u32,
flags,
dimensions: config.dimensions as u32,
created_at: memory::now_secs(),
write_counter: 0,
next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
catalog_start: 0,
catalog_pages: 0,
catalog_len: 0,
vector_count: 0,
m_cost: config.kdf.m_cost,
t_cost: config.kdf.t_cost,
p_cost: config.kdf.p_cost,
salt,
dek_nonce,
wrapped_dek,
index_start: 0,
index_pages: 0,
index_len: 0,
wal_start: 1,
wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
wal_seq: 0,
manifest_start: 0,
manifest_pages: 0,
manifest_len: 0,
seal_nonce: None,
seal_tag: None,
cache_start: 0,
cache_pages: 0,
cache_len: 0,
catalog_gen: 0,
index_gen: 0,
manifest_gen: 0,
cache_gen: 0,
free: RunPtr::default(),
keys: RunPtr::default(),
bm25: RunPtr::default(),
index_delta: RunPtr::default(),
catalog_delta: RunPtr::default(),
};
let mut pager = Pager::new(file, dek, 0, version);
let mut page = header.to_page();
header.apply_seal(&mut page, pager.dek())?;
pager.write_raw(0, &page)?;
pager.sync()?;
Ok(Mnemo {
pager,
header,
catalog: Vec::new(),
index: IdIndex::default(),
path,
dimensions: config.dimensions,
kdf: config.kdf,
dirty_catalog: false,
ann: None,
dirty_index: false,
index_persist: IndexPersist::default(),
ann_unloaded: false,
manifest: Vec::new(),
max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
{
V9_DEFAULT_MAX_SNAPSHOTS
} else {
config.max_snapshots
},
read_only: false,
cache_dir: Vec::new(),
cache_idx: DirectoryIndex::new(),
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
free: FreeList::default(),
pending_free: Vec::new(),
free_dirty: false,
key_slots: KeySlotRun::default(),
dirty_keys: false,
gen_leased: false,
col: VecColumn::default(),
recs: RecPages::default(),
bm25: Bm25State::default(),
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain: Vec::new(),
delta_rows: 0,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
Self::open_secret(path, passphrase.as_bytes())
}
pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
Self::open_secret(path, key)
}
fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
let path: PathBuf = path.into();
let mut file = open_with_lock(&path, true, false)?;
let mut hbuf = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
let mut header = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(e) => {
let scan = scan_header_recovery_candidates(&mut file, secret)?;
let frames = scan.ok_or(e)?;
for (page_no, bytes) in &frames {
if bytes.len() != PAGE_SIZE {
return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
}
file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
file.write_all(bytes)?;
}
file.sync_all()?;
crate::bump_fsync();
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
Header::from_page(&hbuf)?
}
};
if let Some(frames) = wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)? {
for (page_no, bytes) in &frames {
if bytes.len() != PAGE_SIZE {
return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
}
file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
file.write_all(bytes)?;
}
file.sync_all()?;
crate::bump_fsync();
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
header = Header::from_page(&hbuf)?;
}
let kdf = KdfParams {
m_cost: header.m_cost,
t_cost: header.t_cost,
p_cost: header.p_cost,
};
let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
header.validate_seal(&dek)?;
let on_disk_version = header.version;
let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
let dimensions = header.dimensions as usize;
let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
(
load_catalog(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
header.catalog_gen,
)?,
false,
)
} else if on_disk_version == 4 {
let v4 = load_catalog_v4(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
)?;
(migrate_v4_catalog(&mut pager, v4)?, true)
} else {
return Err(MnemoError::UnsupportedVersion(on_disk_version));
};
let (catalog, delta_chain, delta_rows) =
apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
let index = build_id_index(&catalog);
let ann = if on_disk_version < 6 {
load_index(
&mut pager,
header.index_start,
header.index_pages,
header.index_len,
header.index_gen,
dimensions,
)?
} else {
None
};
let ann_unloaded = ann.is_none() && header.index_pages > 0;
let mut manifest = load_manifest(
&mut pager,
header.manifest_start,
header.manifest_pages,
header.manifest_len,
header.manifest_gen,
)?;
let migrated_pages_to_v6 = on_disk_version < 6;
if migrated_pages_to_v6 {
for e in &catalog {
if e.deleted {
continue;
}
for i in 0..e.page_count as u64 {
let payload = pager.read_page(e.start_page + i, 0)?;
pager.write_page(e.start_page + i, 0, &payload)?;
}
}
pager.set_version(6);
}
let migrated_header_to_v7 = on_disk_version < 7;
if migrated_header_to_v7 {
pager.set_version(7);
}
let migrated_header_to_v8 = on_disk_version < 8;
if migrated_header_to_v8 {
pager.set_version(VERSION);
}
let migrated = migrated_from_v4
|| migrated_pages_to_v6
|| migrated_header_to_v7
|| migrated_header_to_v8;
if migrated {
header.version = VERSION;
manifest.clear();
}
let dirty_index = migrated_pages_to_v6 && ann.is_some();
let cache_dir = load_cache_directory(
&mut pager,
header.cache_start,
header.cache_pages,
header.cache_len,
header.cache_gen,
)?;
let cache_idx = DirectoryIndex::build(&cache_dir);
let free = load_free_list(&mut pager, header.free)?;
let key_slots = load_key_slots(&mut pager, header.keys)?;
let col = VecColumn::from_catalog(&catalog);
let recs = RecPages::from_catalog(&catalog);
let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
let max_snapshots = if header.version >= VERSION_V9 {
V9_DEFAULT_MAX_SNAPSHOTS
} else {
DEFAULT_MAX_SNAPSHOTS
};
Ok(Mnemo {
pager,
header,
catalog,
index,
path,
dimensions,
kdf,
dirty_catalog: migrated,
ann,
dirty_index,
index_persist: IndexPersist {
forced: dirty_index,
..IndexPersist::default()
},
ann_unloaded,
manifest,
max_snapshots,
read_only: false,
cache_dir,
cache_idx,
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
free,
pending_free: Vec::new(),
free_dirty: false,
key_slots,
dirty_keys: false,
gen_leased: false,
col,
recs,
bm25: Bm25State {
manifest: bm25_manifest,
..Bm25State::default()
},
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain,
delta_rows,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: false,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
Self::open_read_only_secret(path, passphrase.as_bytes())
}
pub fn open_read_only_with_key(
path: impl Into<PathBuf>,
key: &[u8; crypto::KEY_LEN],
) -> Result<Mnemo> {
Self::open_read_only_secret(path, key)
}
fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
let path: PathBuf = path.into();
let mut file = open_with_lock(&path, false, false)?;
let mut hbuf = [0u8; PAGE_SIZE];
file.seek(SeekFrom::Start(0))?;
file.read_exact(&mut hbuf)?;
let header = match Header::from_page(&hbuf) {
Ok(h) => h,
Err(parse_err) => {
let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
return Err(match (parse_err, recoverable) {
(_, Some(_)) => MnemoError::NeedsWriteOpen {
reason: "torn header page needs recovery from WAL",
},
(e, None) => e,
});
}
};
if header.version < VERSION {
return Err(MnemoError::NeedsWriteOpen {
reason: "pending format-version migration",
});
}
if wal::recover(
&mut file,
header.wal_start,
header.wal_pages,
header.wal_seq,
)?
.is_some()
{
return Err(MnemoError::NeedsWriteOpen {
reason: "pending WAL recovery",
});
}
let kdf = KdfParams {
m_cost: header.m_cost,
t_cost: header.t_cost,
p_cost: header.p_cost,
};
let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
header.validate_seal(&dek)?;
let mut pager = Pager::new(file, dek, header.write_counter, header.version);
let dimensions = header.dimensions as usize;
let catalog = load_catalog(
&mut pager,
header.catalog_start,
header.catalog_pages,
header.catalog_len,
header.catalog_gen,
)?;
let (catalog, delta_chain, delta_rows) =
apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
let index = build_id_index(&catalog);
let ann: Option<IvfPqIndex> = None;
let ann_unloaded = header.index_pages > 0;
let manifest = load_manifest(
&mut pager,
header.manifest_start,
header.manifest_pages,
header.manifest_len,
header.manifest_gen,
)?;
let cache_dir = load_cache_directory(
&mut pager,
header.cache_start,
header.cache_pages,
header.cache_len,
header.cache_gen,
)?;
let cache_idx = DirectoryIndex::build(&cache_dir);
let free = load_free_list(&mut pager, header.free)?;
let key_slots = load_key_slots(&mut pager, header.keys)?;
let col = VecColumn::from_catalog(&catalog);
let recs = RecPages::from_catalog(&catalog);
let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
let max_snapshots = if header.version >= VERSION_V9 {
V9_DEFAULT_MAX_SNAPSHOTS
} else {
DEFAULT_MAX_SNAPSHOTS
};
Ok(Mnemo {
pager,
header,
catalog,
index,
path,
dimensions,
kdf,
dirty_catalog: false,
ann,
dirty_index: false,
index_persist: IndexPersist::default(),
ann_unloaded,
manifest,
max_snapshots,
read_only: true,
cache_dir,
cache_idx,
dirty_cache: false,
cache_budgets: std::collections::HashMap::new(),
cache_flush_policy: CacheFlushPolicy::default(),
cache_batch: BatchState::default(),
cache_counters: CounterState::default(),
embedder: None,
fsyncs_last_flush: 0,
agent_set: None,
key_index: None,
superseded: None,
layers: None,
turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
free,
pending_free: Vec::new(),
free_dirty: false,
key_slots,
dirty_keys: false,
gen_leased: false,
col,
recs,
bm25: Bm25State {
manifest: bm25_manifest,
..Bm25State::default()
},
kw: None,
index_layout_override: None,
dirty_rows: std::collections::BTreeSet::new(),
delta_chain,
delta_rows,
catalog_rewrite: false,
index_chain: Vec::new(),
index_chain_ops: 0,
index_rebase: false,
index_delta_ops: None,
snapshot_cap_explicit: false,
direct_writes: false,
vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
})
}
pub fn set_turn_compress_after(&mut self, secs: u64) {
self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
}
pub fn set_max_snapshots(&mut self, max: usize) {
self.max_snapshots = max;
self.snapshot_cap_explicit = true;
}
pub fn dimensions(&self) -> usize {
self.dimensions
}
pub fn len(&self) -> usize {
self.catalog.iter().filter(|e| !e.deleted).count()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
fn is_v9(&self) -> bool {
self.header.version >= VERSION_V9
}
fn write_gen(&self) -> u64 {
if self.is_v9() {
self.header.wal_seq + 1
} else {
0
}
}
fn min_snapshot_txn(&self) -> u64 {
self.manifest
.iter()
.map(|s| s.txn_id)
.min()
.unwrap_or(u64::MAX)
}
fn alloc_pages(&mut self, n: u64) -> u64 {
if self.is_v9() {
let committed = self.header.wal_seq;
let min_snap = self.min_snapshot_txn();
if let Some(p) = self.free.alloc(n, committed, min_snap) {
self.free_dirty = true;
return p;
}
}
let p = self.header.next_page;
self.header.next_page += n;
p
}
fn release_pages(&mut self, page: u64, n: u64) {
if self.is_v9() && n > 0 && page != 0 {
self.pending_free.push((page, n));
}
}
fn ensure_bm25(&mut self) -> Result<()> {
self.ensure_bm25_stats();
self.ensure_bm25_dicts()
}
fn ensure_bm25_stats(&mut self) {
if self.bm25.stats.is_none() {
let mut docs = std::collections::HashMap::new();
let mut total = 0u64;
for (i, e) in self.catalog.iter().enumerate() {
if e.deleted {
continue;
}
if let Some(r) = e.v9 {
if r.doc != 0 && bm25_kind(r.kind) {
docs.insert(r.doc, (i, r.doc_len));
total += r.doc_len as u64;
}
}
}
let mut stats = DocStats {
docs,
avgdl: 1.0,
total,
};
stats.refresh();
self.bm25.stats = Some(stats);
}
}
fn ensure_bm25_dicts(&mut self) -> Result<()> {
if self.bm25.dicts.is_none() {
let mut dicts = Vec::with_capacity(self.bm25.manifest.segments.len());
for &t in &self.bm25.manifest.segments {
dicts.push(load_segment_dict(&mut self.pager, tuple_ptr(t))?);
}
self.bm25.dicts = Some(dicts);
}
Ok(())
}
fn bm25_scores(
&mut self,
query: &str,
k1: f32,
b: f32,
) -> Result<std::collections::HashMap<usize, f32>> {
let mut scores: std::collections::HashMap<usize, f32> = std::collections::HashMap::new();
if !self.is_v9() {
return Ok(scores);
}
self.ensure_bm25()?;
let terms: std::collections::BTreeSet<String> = text::tokenize(query).into_iter().collect();
let segs: Vec<RunPtr> = self
.bm25
.manifest
.segments
.iter()
.map(|&t| tuple_ptr(t))
.collect();
let stats = self.bm25.stats.as_ref().expect("loaded above");
let dicts = self.bm25.dicts.as_ref().expect("loaded above");
let n = stats.docs.len();
for term in &terms {
let mut posts: Vec<(u32, u8)> = Vec::new();
for (si, seg) in segs.iter().enumerate() {
posts.extend(segment_postings(&mut self.pager, *seg, &dicts[si], term)?);
}
for (doc, tfs) in &self.bm25.pending {
if let Ok(i) = tfs.binary_search_by(|(t, _)| t.as_str().cmp(term)) {
posts.push((*doc, tfs[i].1));
}
}
posts.retain(|(d, _)| stats.docs.contains_key(d));
if posts.is_empty() {
continue;
}
let idf = text::idf(n, posts.len());
for (d, tf) in posts {
let (row, len) = stats.docs[&d];
*scores.entry(row).or_default() += text::bm25(tf, idf, len, stats.avgdl, k1, b);
}
}
Ok(scores)
}
fn bm25_next_segment(&mut self) -> Result<Option<(Vec<u8>, Option<usize>)>> {
if !self.is_v9() || self.bm25.pending.is_empty() {
return Ok(None);
}
let segs: Vec<RunPtr> = self
.bm25
.manifest
.segments
.iter()
.map(|&t| tuple_ptr(t))
.collect();
self.ensure_bm25_stats();
let written: u64 = self.bm25.manifest.seg_docs.iter().map(|&d| d as u64).sum();
let live_docs = self.bm25.stats.as_ref().map_or(0, |s| s.docs.len()) as u64;
let mostly_dead = !segs.is_empty() && written > 64 && live_docs * 2 < written;
if segs.len() < BM25_MAX_SEGMENTS && !mostly_dead {
return Ok(Some((text::build_segment(&self.bm25.pending)?, None)));
}
let small: u64 = segs.iter().skip(1).map(|p| p.len).sum();
let from = if mostly_dead || small * 10 > segs[0].len {
0
} else {
1
};
let live: std::collections::HashSet<u32> = self
.bm25
.stats
.as_ref()
.expect("loaded above")
.docs
.keys()
.copied()
.collect();
let mut by_doc: std::collections::BTreeMap<u32, Vec<(String, u8)>> =
std::collections::BTreeMap::new();
for seg in &segs[from..] {
let bytes = read_run_bytes(&mut self.pager, seg.start, seg.pages, seg.len, seg.gen)?;
let n = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
let dict: Vec<text::DictEntry> = rmp_serde::from_slice(&bytes[4..4 + n])
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
let post = &bytes[4 + n..];
for e in dict {
let range = e.off as usize..(e.off + e.len) as usize;
let list = text::decode_postings(post.get(range).ok_or_else(|| {
MnemoError::Invalid("BM25 dict entry past its postings".into())
})?)?;
for (d, tf) in list {
if live.contains(&d) {
by_doc.entry(d).or_default().push((e.term.clone(), tf));
}
}
}
}
for (d, tfs) in &self.bm25.pending {
if live.contains(d) {
by_doc.insert(*d, tfs.clone());
}
}
let docs: Vec<(u32, Vec<(String, u8)>)> = by_doc.into_iter().collect();
self.bm25.merged_docs = docs.len() as u32;
Ok(Some((text::build_segment(&docs)?, Some(from))))
}
fn index_gen_now(&self) -> u64 {
self.header.index_gen.max(self.header.index_delta.gen)
}
fn plan_index_v9(&mut self) -> bool {
self.index_delta_ops = None;
let Some(ann) = self.ann.as_mut() else {
return self.dirty_index;
};
let ops = ann.take_log();
let n = ann.len();
if self.index_rebase {
return true;
}
if ops.is_empty() {
return false;
}
if self.index_chain.len() >= 64 || (self.index_chain_ops + ops.len()) * 4 > n.max(256) {
return true;
}
self.index_delta_ops = Some(ops);
true
}
fn touch(&mut self, idx: usize) {
if self.is_v9() {
self.dirty_rows.insert(idx);
}
}
fn catalog_as_delta(&self) -> bool {
self.is_v9()
&& !self.catalog_rewrite
&& self.header.catalog_pages > 0
&& self.delta_chain.len() < 64
&& (self.delta_rows + self.dirty_rows.len()) * 4 <= self.catalog.len()
}
fn compact_tombstones(&mut self) {
let dead = self.catalog.iter().filter(|e| e.deleted).count();
if dead == 0 || dead * 4 <= self.catalog.len() {
return;
}
self.catalog.retain(|e| !e.deleted);
self.catalog.shrink_to_fit();
self.index = build_id_index(&self.catalog);
self.col = VecColumn::from_catalog(&self.catalog);
self.recs = RecPages::from_catalog(&self.catalog);
self.dirty_rows.clear();
self.bm25.stats = None;
}
fn float_vectors(&self) -> bool {
self.is_v9() && self.header.flags & FLAG_FLOAT_VECTORS != 0
}
fn stage_vector(&mut self, idx: usize, v: &[f32]) {
let (h, norm) = veccol::quantize(v);
let d = self.dimensions;
if self.col.resident {
let rows = self.catalog.len();
let fits = rows * veccol::entry_len(d) <= self.vector_cache_budget;
if fits {
if self.col.data.len() < rows * d {
self.col.data.resize(rows * d, 0);
self.col.norms.resize(rows, 0.0);
self.col.have.resize(rows, false);
}
self.col.data[idx * d..(idx + 1) * d].copy_from_slice(&h);
self.col.norms[idx] = norm;
self.col.have[idx] = true;
} else {
self.col.data = Vec::new();
self.col.norms = Vec::new();
self.col.have = Vec::new();
self.col.resident = false;
}
}
self.col.hot.insert(idx, (h, norm));
self.col.pending.push(idx);
self.touch(idx);
if let Some(r) = self.catalog[idx].v9.as_mut() {
r.vec_page = 0;
r.vec_slot = 0;
r.vec_gen = 0;
}
}
fn kill_vector(&mut self, idx: usize, row: &CatalogEntry) {
self.col.hot.remove(&idx);
let Some(r) = row.v9 else { return };
if r.vec_page == 0 {
return;
}
let empty = match self.col.page_rows.get_mut(&r.vec_page) {
Some(rows) => {
rows.retain(|&i| i != idx);
rows.is_empty()
}
None => false,
};
if empty {
self.col.page_rows.remove(&r.vec_page);
self.release_pages(r.vec_page, 1);
}
}
fn rerank_for(&self, ann: &IvfPqIndex, req: &RecallRequest) -> Option<usize> {
if req.n_rerank.is_some() || !self.is_v9() {
return req.n_rerank;
}
let (_, auto) = ann.effective_defaults();
Some(auto.max(V9_RERANK_PER_HIT.saturating_mul(req.top_k)))
}
fn turn_is_cold(&self, created_at: i64) -> bool {
self.turn_compress_after == 0
|| created_at <= memory::now_secs().saturating_sub(self.turn_compress_after)
}
fn unsettled_turn(&self, idx: usize) -> bool {
let e = &self.catalog[idx];
!e.deleted
&& e.v9
.is_some_and(|r| r.kind == ROW_KIND_TURN && r.flags & ROW_SETTLED == 0)
}
fn settle_turn(&mut self, idx: usize, bytes: Vec<u8>) -> Vec<u8> {
let out = if bytes.starts_with(&ZSTD_MAGIC) {
bytes
} else {
compress_turn_record(bytes)
};
let e = &mut self.catalog[idx];
e.len = out.len() as u32;
if let Some(r) = e.v9.as_mut() {
r.flags |= ROW_SETTLED;
}
self.touch(idx);
self.dirty_catalog = true;
out
}
fn settle_cold_turns(&mut self) -> Result<()> {
if !self.is_v9() || !self.dirty_catalog {
return Ok(());
}
let picked: Vec<usize> = (0..self.catalog.len())
.filter(|&i| {
self.unsettled_turn(i)
&& self.catalog[i]
.v9
.is_some_and(|r| self.turn_is_cold(r.created_at))
})
.take(COLD_TURNS_PER_COMMIT)
.collect();
for idx in picked {
let old = self.catalog[idx].clone();
let bytes = self.read_record(&old)?;
let before = bytes.len();
let out = self.settle_turn(idx, bytes);
if out.len() == before {
continue;
}
self.kill_record(idx, &old);
let gen = self.write_gen();
let e = &mut self.catalog[idx];
if out.len() + REC_SLOT_BYTES <= REC_PAGE_CAP {
e.start_page = 0;
e.page_count = 1;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = 0;
}
self.recs.hot.insert(idx, out);
self.recs.pending.push(idx);
} else {
let (start, pc, gen) = self.write_record(&out)?;
let e = &mut self.catalog[idx];
e.start_page = start;
e.page_count = pc;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = REC_SPAN;
}
}
}
Ok(())
}
fn kill_record(&mut self, idx: usize, row: &CatalogEntry) {
self.recs.hot.remove(&idx);
let Some(r) = row.v9 else { return };
if r.rec_slot == REC_SPAN {
self.release_pages(row.start_page, row.page_count as u64);
return;
}
if row.start_page == 0 {
return;
}
let empty = match self.recs.page_rows.get_mut(&row.start_page) {
Some(rows) => {
rows.retain(|&i| i != idx);
rows.is_empty()
}
None => false,
};
if empty {
self.recs.page_rows.remove(&row.start_page);
self.release_pages(row.start_page, 1);
}
}
fn pack_records(&mut self) -> Result<()> {
if !self.is_v9() || self.recs.pending.is_empty() {
return Ok(());
}
let mut sparse: Vec<(u64, usize)> = self
.recs
.page_rows
.iter()
.filter(|(_, rows)| {
let live: usize = rows
.iter()
.map(|&i| self.catalog[i].len as usize + REC_SLOT_BYTES)
.sum();
live * 4 < REC_PAGE_CAP * 3
})
.map(|(&p, rows)| (p, rows.len()))
.collect();
sparse.sort_unstable();
let sparse = vacuum_pick(sparse);
let mut order: Vec<usize> = Vec::new();
let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
for p in sparse {
let rows = self.recs.page_rows.remove(&p).unwrap_or_default();
if let Some(&first) = rows.first() {
let page = self.pager.read_page_uncached(p, self.catalog[first].gen)?;
for idx in rows {
let slot = self.catalog[idx].v9.map_or(0, |r| r.rec_slot);
let mut b = record_slot(&page, slot)?;
if self.unsettled_turn(idx) {
b = self.settle_turn(idx, b);
}
self.recs.hot.insert(idx, b);
if seen.insert(idx) {
order.push(idx);
}
}
}
self.release_pages(p, 1);
}
let pending = std::mem::take(&mut self.recs.pending);
for idx in pending {
if self.recs.hot.contains_key(&idx) && seen.insert(idx) {
order.push(idx);
}
}
let gen = self.write_gen();
let mut i = 0;
while i < order.len() {
let mut used = 0usize;
let mut j = i;
while j < order.len() {
let l = self.recs.hot[&order[j]].len() + REC_SLOT_BYTES;
if j > i && used + l > REC_PAGE_CAP {
break;
}
used += l;
j += 1;
}
let page = self.alloc_pages(1);
let bytes = {
let recs: Vec<&[u8]> = order[i..j]
.iter()
.map(|k| self.recs.hot[k].as_slice())
.collect();
encode_record_page(gen, &recs)
};
self.pager.write_page(page, gen, &bytes)?;
for (slot, &k) in order[i..j].iter().enumerate() {
self.touch(k);
let e = &mut self.catalog[k];
e.start_page = page;
e.page_count = 1;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = slot as u16;
}
}
self.recs.page_rows.insert(page, order[i..j].to_vec());
i = j;
}
for k in order {
self.recs.hot.remove(&k);
}
self.dirty_catalog = true;
Ok(())
}
fn column_entry(&mut self, e: &CatalogEntry) -> Result<(Vec<u16>, f32)> {
let d = self.dimensions;
if let Some(idx) = self.index.get(e.id, &self.catalog) {
if let Some((h, n)) = self.col.hot.get(&idx) {
return Ok((h.clone(), *n));
}
if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
return Ok((
self.col.data[idx * d..(idx + 1) * d].to_vec(),
self.col.norms[idx],
));
}
}
let r =
e.v9.ok_or_else(|| MnemoError::Invalid("v9 row without a vector entry".into()))?;
if r.vec_page == 0 {
return Err(MnemoError::Invalid("vector entry not written yet".into()));
}
let page = self.pager.read_page(r.vec_page, r.vec_gen)?;
veccol::decode_entry(&page, d, r.vec_slot as usize)
}
fn ensure_column(&mut self) -> Result<()> {
if !self.is_v9() || self.col.resident {
return Ok(());
}
let d = self.dimensions;
let rows = self.catalog.len();
if rows * veccol::entry_len(d) > self.vector_cache_budget {
return Ok(());
}
self.col.data = vec![0u16; rows * d];
self.col.norms = vec![0f32; rows];
self.col.have = vec![false; rows];
let hot: Vec<(usize, Vec<u16>, f32)> = self
.col
.hot
.iter()
.filter(|(&i, _)| i < rows)
.map(|(&i, (h, n))| (i, h.clone(), *n))
.collect();
for (i, h, n) in hot {
self.col.data[i * d..(i + 1) * d].copy_from_slice(&h);
self.col.norms[i] = n;
self.col.have[i] = true;
}
self.col.resident = true;
Ok(())
}
fn fill_column_page(&mut self, idx: usize) -> Result<()> {
let Some(r) = self.catalog[idx].v9 else {
return Ok(());
};
if r.vec_page == 0 {
return Ok(());
}
let d = self.dimensions;
let bytes = self.pager.read_page_uncached(r.vec_page, r.vec_gen)?;
let rows = self
.col
.page_rows
.get(&r.vec_page)
.cloned()
.unwrap_or_default();
for i in rows.into_iter().chain(std::iter::once(idx)) {
if i >= self.col.have.len() || self.col.have[i] {
continue;
}
let Some(ri) = self.catalog[i].v9 else {
continue;
};
if ri.vec_page != r.vec_page {
continue;
}
let n = veccol::decode_entry_into(
&bytes,
d,
ri.vec_slot as usize,
&mut self.col.data[i * d..(i + 1) * d],
)?;
self.col.norms[i] = n;
self.col.have[i] = true;
}
Ok(())
}
fn pack_vectors(&mut self) -> Result<()> {
if !self.is_v9() || self.col.pending.is_empty() {
return Ok(());
}
let d = self.dimensions;
let per = veccol::per_page(d);
let mut sparse: Vec<(u64, usize)> = self
.col
.page_rows
.iter()
.filter(|(_, rows)| rows.len() * 4 < per * 3)
.map(|(&p, rows)| (p, rows.len()))
.collect();
sparse.sort_unstable();
let sparse = vacuum_pick(sparse);
let mut order: Vec<usize> = Vec::new();
let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
for p in sparse {
let rows = self.col.page_rows.remove(&p).unwrap_or_default();
for idx in rows {
let e = self.catalog[idx].clone();
let v = self.column_entry(&e)?;
self.col.hot.insert(idx, v);
if seen.insert(idx) {
order.push(idx);
}
}
self.release_pages(p, 1);
}
let pending = std::mem::take(&mut self.col.pending);
for idx in pending {
if self.col.hot.contains_key(&idx) && seen.insert(idx) {
order.push(idx);
}
}
let gen = self.write_gen();
for chunk in order.chunks(per) {
let page = self.alloc_pages(1);
let bytes = {
let entries: Vec<(&[u16], f32)> = chunk
.iter()
.map(|i| {
let (h, n) = &self.col.hot[i];
(h.as_slice(), *n)
})
.collect();
veccol::encode_page(d, gen, &entries)
};
self.pager.write_page(page, gen, &bytes)?;
for (slot, &i) in chunk.iter().enumerate() {
self.touch(i);
if let Some(r) = self.catalog[i].v9.as_mut() {
r.vec_page = page;
r.vec_slot = slot as u16;
r.vec_gen = gen;
}
}
self.col.page_rows.insert(page, chunk.to_vec());
}
for i in order {
self.col.hot.remove(&i);
}
self.dirty_catalog = true;
Ok(())
}
fn ensure_gen_lease(&mut self) -> Result<()> {
if !self.is_v9() || self.gen_leased || self.read_only {
return Ok(());
}
self.header.wal_seq += 1;
let wal_start = self.header.wal_start;
self.relocate_header_through_wal(wal_start)?;
self.gen_leased = true;
Ok(())
}
fn write_record(&mut self, bytes: &[u8]) -> Result<(u64, u32, u64)> {
self.ensure_gen_lease()?;
let pc = bytes.len().div_ceil(PAYLOAD).max(1);
let start = self.alloc_pages(pc as u64);
let gen = self.write_gen();
for i in 0..pc {
let lo = i * PAYLOAD;
let hi = ((i + 1) * PAYLOAD).min(bytes.len());
self.pager
.write_page(start + i as u64, gen, &bytes[lo..hi])?;
}
Ok((start, pc as u32, gen))
}
fn read_record_run(&mut self, start: u64, pages: u32, len: u32, gen: u64) -> Result<Vec<u8>> {
let mut buf = Vec::with_capacity(len as usize);
for i in 0..pages as u64 {
buf.extend_from_slice(&self.pager.read_page(start + i, gen)?);
}
buf.truncate(len as usize);
Ok(buf)
}
fn read_record(&mut self, e: &CatalogEntry) -> Result<Vec<u8>> {
if let Some(r) = e.v9 {
if r.rec_slot != REC_SPAN {
if e.start_page == 0 {
return self
.index
.get(e.id, &self.catalog)
.and_then(|idx| self.recs.hot.get(&idx).cloned())
.ok_or_else(|| MnemoError::Invalid("record not written yet".into()));
}
let page = self.pager.read_page(e.start_page, e.gen)?;
return record_slot(&page, r.rec_slot);
}
}
let mut buf = Vec::with_capacity(e.len as usize);
for i in 0..e.page_count as u64 {
buf.extend_from_slice(&self.pager.read_page(e.start_page + i, e.gen)?);
}
buf.truncate(e.len as usize);
Ok(buf)
}
fn decode_memory(&self, bytes: &[u8]) -> Result<Memory> {
let plain = if self.is_v9() {
plain_record(bytes)?
} else {
std::borrow::Cow::Borrowed(bytes)
};
rmp_serde::from_slice(&plain).map_err(|err| MnemoError::Serialize(err.to_string()))
}
fn read_memory(&mut self, e: &CatalogEntry) -> Result<Memory> {
let m = self.read_memory_meta(e)?;
self.finish_memory(e, m)
}
fn finish_memory(&mut self, e: &CatalogEntry, mut m: Memory) -> Result<Memory> {
if e.v9.is_some() && !self.float_vectors() {
let (h, _) = self.column_entry(e)?;
m.vector = veccol::dequantize(&h);
}
m.accessed_at = e.accessed_at;
m.access_count = e.access_count;
Ok(m)
}
fn put(&mut self, mut m: Memory) -> Result<Ulid> {
if m.vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: m.vector.len(),
});
}
if embed::is_zero_vector(&m.vector) {
return Err(MnemoError::Invalid(
"all-zero vector refused: it scores 0.0 on every cosine query and \
pollutes recall; check your embedder returned a real embedding"
.into(),
));
}
if m.id == Ulid::nil() {
m.id = Ulid::new();
}
let id_u: u128 = m.id.0;
if self.layers.is_some() {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
let old_entry = self.catalog[idx].clone();
if !old_entry.deleted {
let old = self.read_memory(&old_entry)?;
self.note_removed(&old)?;
}
}
}
let v9 = self.is_v9();
let bytes = if v9 && !self.float_vectors() {
let vector = std::mem::take(&mut m.vector);
let b = rmp_serde::to_vec(&m);
m.vector = vector;
b
} else {
rmp_serde::to_vec(&m)
}
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
let is_turn = v9 && meta_str(&m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN);
let settle =
is_turn && (bytes.len() < TURN_ZSTD_MIN_BYTES || self.turn_is_cold(m.created_at));
let bytes = if settle {
compress_turn_record(bytes)
} else {
bytes
};
let packed = v9 && bytes.len() + REC_SLOT_BYTES <= REC_PAGE_CAP;
let (start, pc, gen) = if packed {
self.ensure_gen_lease()?;
(0, 1, self.write_gen())
} else {
self.write_record(&bytes)?
};
let rec_len = bytes.len();
let mut tags = v9.then(|| row_tags(&m));
if let Some(r) = tags.as_mut() {
if bm25_kind(r.kind) {
let (terms, len) = text::term_freqs(&m.content);
r.doc = self.bm25.manifest.next_doc;
r.doc_len = len;
self.bm25.manifest.next_doc += 1;
self.bm25.pending.push((r.doc, terms));
}
}
let bm25_new = tags.filter(|r| r.doc != 0 && bm25_kind(r.kind));
let (prev_accessed, prev_count) = match self.index.get(id_u, &self.catalog) {
Some(idx) => (
self.catalog[idx].accessed_at,
self.catalog[idx].access_count,
),
None => (m.accessed_at, m.access_count),
};
let entry = CatalogEntry {
id: id_u,
start_page: start,
page_count: pc,
len: rec_len as u32,
deleted: false,
accessed_at: prev_accessed,
access_count: prev_count,
gen,
v9: tags.map(|mut r| {
r.rec_slot = if packed { 0 } else { REC_SPAN };
if settle {
r.flags |= ROW_SETTLED;
}
r
}),
};
match self.index.get(id_u, &self.catalog) {
Some(idx) => {
if !self.catalog[idx].deleted {
let old = self.catalog[idx].clone();
if v9 {
self.kill_vector(idx, &old);
self.kill_record(idx, &old);
} else {
self.release_pages(old.start_page, old.page_count as u64);
}
}
if let (Some(stats), Some(old)) = (self.bm25.stats.as_mut(), self.catalog[idx].v9) {
if !self.catalog[idx].deleted && old.doc != 0 {
stats.remove(old.doc);
}
}
self.catalog[idx] = entry;
self.touch(idx);
}
None => {
self.index.insert(id_u, self.catalog.len(), &self.catalog);
self.touch(self.catalog.len());
self.catalog.push(entry);
}
}
if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), bm25_new) {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
stats.insert(r.doc, idx, r.doc_len);
}
}
self.dirty_catalog = true;
if v9 {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
self.stage_vector(idx, &m.vector);
if packed {
self.recs.hot.insert(idx, bytes);
self.recs.pending.push(idx);
}
}
}
if let Some(ann) = &mut self.ann {
if is_ann_kind(&m) {
ann.add(id_u, &m.vector);
self.dirty_index = true;
self.index_persist.changes += 1;
} else if ann.contains(id_u) {
ann.remove(id_u);
self.dirty_index = true;
self.index_persist.changes += 1;
}
}
self.note_added(&m)?;
if let Some(set) = &mut self.agent_set {
set.insert(m.agent_id.clone());
}
Ok(m.id)
}
pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
let identity = embedder.identity();
if identity.dimensions != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: identity.dimensions,
});
}
let onboarding = self.about()?;
let mut upsert_target: Option<usize> = None;
for (i, m) in onboarding.iter().enumerate() {
if let Some(saw) = embed::read_identity_from_metadata(&m.metadata) {
if saw != identity {
return Err(embed::identity_mismatch_err(&saw, &identity));
}
self.embedder = Some(std::sync::Arc::new(embedder));
return Ok(());
}
if upsert_target.is_none() {
upsert_target = Some(i);
}
}
if !self.read_only {
if let Some(idx) = upsert_target {
let mut m = onboarding[idx].clone();
embed::write_identity_into_metadata(&mut m.metadata, &identity);
self.put(m)?;
self.flush()?;
}
}
self.embedder = Some(std::sync::Arc::new(embedder));
Ok(())
}
pub fn embedder(&self) -> Option<&dyn Embedder> {
self.embedder.as_deref()
}
pub fn embedder_arc(&self) -> Option<std::sync::Arc<dyn Embedder>> {
self.embedder.clone()
}
pub fn remember_text(&mut self, content: &str, memory_type: MemoryType) -> Result<Ulid> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let embedder = self.embedder.as_ref().ok_or_else(|| {
MnemoError::Invalid(
"no embedder registered on this handle; call set_embedder first".into(),
)
})?;
let mut vectors = embedder.embed(&[content])?;
let vector = vectors.pop().ok_or_else(|| {
MnemoError::Invalid("embedder returned no vector for the input text".into())
})?;
if vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: vector.len(),
});
}
let memory = Memory::new(content.to_string(), memory_type, vector);
self.remember(memory)
}
pub fn remember(&mut self, memory: Memory) -> Result<Ulid> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
self.put(memory)
}
pub fn get(&mut self, id: &Ulid) -> Result<Memory> {
let idx = self
.index
.get(id.0, &self.catalog)
.ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
let entry = self.catalog[idx].clone();
if entry.deleted {
return Err(MnemoError::NotFound(id.to_string()));
}
self.read_memory(&entry)
}
pub fn delete(&mut self, id: &Ulid) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let idx = self
.index
.get(id.0, &self.catalog)
.ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
let entry = self.catalog[idx].clone();
if self.layers.is_some() && !entry.deleted {
let old = self.read_memory(&entry)?;
self.note_removed(&old)?;
}
if !self.catalog[idx].deleted {
self.catalog[idx].deleted = true;
self.dirty_catalog = true;
self.touch(idx);
if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), entry.v9) {
if r.doc != 0 {
stats.remove(r.doc);
}
}
if self.is_v9() {
self.kill_record(idx, &entry);
self.kill_vector(idx, &entry);
} else {
self.release_pages(entry.start_page, entry.page_count as u64);
}
}
if let Some(ann) = &mut self.ann {
if ann.contains(id.0) {
ann.remove(id.0);
self.dirty_index = true;
self.index_persist.changes += 1;
}
}
self.agent_set = None;
Ok(())
}
fn note_added(&mut self, m: &Memory) -> Result<()> {
if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
self.warm_thread(thread)?;
}
let id = m.id.0;
if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
sup.insert(id);
} else if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
keys.insert(Self::key_slot_for(m, k), id);
}
}
if let Some(layers) = self.layers.as_mut() {
layers.add(m);
}
Ok(())
}
fn note_removed(&mut self, m: &Memory) -> Result<()> {
if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
self.warm_thread(thread)?;
}
let id = m.id.0;
if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
sup.remove(&id);
if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
let slot = Self::key_slot_for(m, k);
if keys.get(&slot) == Some(&id) {
keys.remove(&slot);
}
}
}
if let Some(layers) = self.layers.as_mut() {
layers.remove(m);
}
Ok(())
}
fn warm_thread(&mut self, thread: &str) -> Result<()> {
self.warm_thread_decoded(thread).map(|_| ())
}
fn warm_thread_decoded(
&mut self,
thread: &str,
) -> Result<std::collections::HashMap<u128, Memory>> {
let mut decoded = std::collections::HashMap::new();
let h = fnv1a32(&[thread.as_bytes()]);
let Some(ids) = self.layers.as_mut().and_then(|l| l.cold_turns.remove(&h)) else {
return Ok(decoded);
};
for e in self.live_entries(ids) {
let m = self.read_memory_meta(&e)?;
if let Some(layers) = self.layers.as_mut() {
layers.add(&m);
}
decoded.insert(e.id, m);
}
Ok(decoded)
}
fn read_memory_meta(&mut self, e: &CatalogEntry) -> Result<Memory> {
let bytes = self.read_record(e)?;
self.decode_memory(&bytes)
}
fn live_entries(&self, ids: impl IntoIterator<Item = u128>) -> Vec<CatalogEntry> {
ids.into_iter()
.filter_map(|id| {
self.index
.get(id, &self.catalog)
.map(|i| self.catalog[i].clone())
})
.filter(|e| !e.deleted)
.collect()
}
fn essence_entries(&mut self) -> Result<Vec<CatalogEntry>> {
self.ensure_key_caches()?;
let ids: Vec<u128> = self
.layers
.as_ref()
.map(|l| l.essence.iter().copied().collect())
.unwrap_or_default();
Ok(self.live_entries(ids))
}
fn thread_entries(&mut self, thread: &str) -> Result<Vec<CatalogEntry>> {
Ok(self.thread_entries_decoded(thread)?.0)
}
fn thread_entries_decoded(
&mut self,
thread: &str,
) -> Result<(Vec<CatalogEntry>, std::collections::HashMap<u128, Memory>)> {
self.ensure_key_caches()?;
let decoded = self.warm_thread_decoded(thread)?;
let ids: Vec<u128> = self
.layers
.as_ref()
.and_then(|l| l.threads.get(thread))
.map(|tc| tc.records.iter().copied().collect())
.unwrap_or_default();
Ok((self.live_entries(ids), decoded))
}
fn key_slot_for(memory: &Memory, key: &str) -> crate::store::KeySlot {
let scope_owner = match memory.scope {
Scope::Private => Some(memory.agent_id.clone()),
Scope::Shared => None,
};
crate::store::KeySlot {
scope_owner,
key: key.to_string(),
}
}
fn ensure_key_caches(&mut self) -> Result<()> {
if self.key_index.is_some() && self.superseded.is_some() && self.layers.is_some() {
return Ok(());
}
let entries: Vec<CatalogEntry> = self
.catalog
.iter()
.filter(|e| !e.deleted)
.cloned()
.collect();
let mut keys: std::collections::HashMap<crate::store::KeySlot, u128> =
std::collections::HashMap::new();
let mut superseded: std::collections::HashSet<u128> = std::collections::HashSet::new();
let mut layers = LayerCache::default();
let mut non_facts: Vec<u128> = Vec::new();
for e in &entries {
if let Some(r) = e.v9 {
if r.kind == ROW_KIND_TURN
&& r.thread_hash != 0
&& r.key_hash == 0
&& r.flags & ROW_SUPERSEDED == 0
{
layers
.cold_turns
.entry(r.thread_hash)
.or_default()
.push(e.id);
non_facts.push(e.id);
continue;
}
}
let m = self.read_memory_meta(e)?;
layers.add(&m);
if !is_ann_kind(&m) {
non_facts.push(e.id);
}
if m.metadata.get(MNEMO_META_SUPERSEDED_BY).is_some() {
superseded.insert(e.id);
continue;
}
if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
let slot = Self::key_slot_for(&m, k);
keys.entry(slot)
.and_modify(|cur| {
if *cur < e.id {
*cur = e.id;
}
})
.or_insert(e.id);
}
}
self.key_index = Some(keys);
self.superseded = Some(superseded);
self.layers = Some(layers);
if let Some(ann) = &mut self.ann {
let mut pruned = 0usize;
for id in non_facts {
if ann.contains(id) {
ann.remove(id);
pruned += 1;
}
}
if pruned > 0 {
self.dirty_index = true;
self.index_persist.changes += pruned;
}
}
Ok(())
}
fn is_identical_keyed_write(incoming: &Memory, live: &Memory) -> bool {
if incoming.content != live.content {
return false;
}
if incoming.memory_type != live.memory_type {
return false;
}
if (incoming.importance - live.importance).abs() > f32::EPSILON {
return false;
}
if incoming.vector != live.vector {
return false;
}
let strip = |m: &Map<String, Value>| -> Map<String, Value> {
m.iter()
.filter(|(k, _)| !k.starts_with(crate::memory::MNEMO_META_NS))
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
};
strip(&incoming.metadata) == strip(&live.metadata)
}
pub fn remember_keyed(
&mut self,
mut memory: Memory,
key: &str,
) -> Result<(Ulid, WriteOutcome)> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if key.is_empty() {
return Err(MnemoError::Invalid(
"remember_keyed: 'key' must not be empty".into(),
));
}
self.ensure_key_caches()?;
let slot = Self::key_slot_for(&memory, key);
let existing = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
memory
.metadata
.insert(MNEMO_META_KEY.into(), Value::String(key.to_string()));
if let Some(prev_id_u) = existing {
let prev_idx = self
.index
.get(prev_id_u, &self.catalog)
.ok_or_else(|| MnemoError::Invalid("key_index drift".into()))?;
let prev_entry = self.catalog[prev_idx].clone();
let prev_mem = self.read_memory(&prev_entry)?;
if Self::is_identical_keyed_write(&memory, &prev_mem) {
return Ok((Ulid(prev_id_u), WriteOutcome::Unchanged));
}
let now = memory::now_secs();
memory.id = Ulid::nil(); memory.metadata.insert(
MNEMO_META_SUPERSEDES.into(),
Value::String(Ulid(prev_id_u).to_string()),
);
let new_id = self.put(memory)?;
let mut prev_mut = prev_mem;
prev_mut.metadata.insert(
MNEMO_META_SUPERSEDED_BY.into(),
Value::String(new_id.to_string()),
);
prev_mut
.metadata
.insert(MNEMO_META_INVALID_AT.into(), Value::Number(now.into()));
self.put(prev_mut)?;
if let Some(k) = self.key_index.as_mut() {
k.insert(slot, new_id.0);
}
if let Some(s) = self.superseded.as_mut() {
s.insert(prev_id_u);
}
return Ok((new_id, WriteOutcome::Superseded));
}
memory.id = Ulid::nil();
let new_id = self.put(memory)?;
if let Some(k) = self.key_index.as_mut() {
k.insert(slot, new_id.0);
}
Ok((new_id, WriteOutcome::Added))
}
pub fn remember_keyed_text(
&mut self,
content: &str,
memory_type: MemoryType,
key: &str,
) -> Result<(Ulid, WriteOutcome)> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let embedder = self.embedder.as_ref().ok_or_else(|| {
MnemoError::Invalid(
"no embedder registered on this handle; call set_embedder first".into(),
)
})?;
let mut vectors = embedder.embed(&[content])?;
let vector = vectors.pop().ok_or_else(|| {
MnemoError::Invalid("embedder returned no vector for the input text".into())
})?;
if vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: vector.len(),
});
}
let memory = Memory::new(content.to_string(), memory_type, vector);
self.remember_keyed(memory, key)
}
pub fn update(&mut self, id: &Ulid, patch: MemoryPatch) -> Result<WriteOutcome> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if patch.content.is_some() && patch.vector.is_none() {
return Err(MnemoError::Invalid(
"update: a content change needs a matching vector (or call update_text \
to re-embed through the registered embedder)"
.into(),
));
}
let idx = self
.index
.get(id.0, &self.catalog)
.ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
if self.catalog[idx].deleted {
return Err(MnemoError::NotFound(id.to_string()));
}
let entry = self.catalog[idx].clone();
let existing = self.read_memory(&entry)?;
let mut cand = existing.clone();
if let Some(c) = &patch.content {
cand.content = c.clone();
}
if let Some(v) = &patch.vector {
if v.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: v.len(),
});
}
cand.vector = v.clone();
}
if let Some(i) = patch.importance {
cand.importance = i.clamp(0.0, 1.0);
}
if let Some(mt) = patch.memory_type {
cand.memory_type = mt;
}
if let Some(new_meta) = patch.metadata {
let reserved: Vec<(String, Value)> = existing
.metadata
.iter()
.filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
cand.metadata = new_meta;
for (k, v) in reserved {
cand.metadata.insert(k, v);
}
}
if let Some(ttl) = patch.ttl_secs {
cand.ttl_secs = ttl;
}
if cand.content == existing.content
&& cand.vector == existing.vector
&& (cand.importance - existing.importance).abs() <= f32::EPSILON
&& cand.memory_type == existing.memory_type
&& cand.metadata == existing.metadata
&& cand.ttl_secs == existing.ttl_secs
{
return Ok(WriteOutcome::Unchanged);
}
let now = memory::now_secs();
cand.metadata
.insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
cand.id = existing.id;
cand.created_at = existing.created_at;
self.put(cand)?;
Ok(WriteOutcome::Updated)
}
pub fn update_text(
&mut self,
id: &Ulid,
new_text: &str,
mut patch: MemoryPatch,
) -> Result<WriteOutcome> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let embedder = self.embedder.as_ref().ok_or_else(|| {
MnemoError::Invalid(
"no embedder registered on this handle; call set_embedder first".into(),
)
})?;
let mut vectors = embedder.embed(&[new_text])?;
let vector = vectors.pop().ok_or_else(|| {
MnemoError::Invalid("embedder returned no vector for the input text".into())
})?;
if vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: vector.len(),
});
}
patch.content = Some(new_text.to_string());
patch.vector = Some(vector);
self.update(id, patch)
}
pub fn history(&mut self, key: &str, agent_id: Option<&str>) -> Result<Vec<Memory>> {
let scope = match agent_id {
Some(a) => KeyScope::Private(a.to_string()),
None => KeyScope::private_default(),
};
self.history_in(key, scope)
}
pub fn history_in(&mut self, key: &str, scope: KeyScope) -> Result<Vec<Memory>> {
self.ensure_key_caches()?;
let slot = crate::store::KeySlot {
scope_owner: match scope {
KeyScope::Private(a) => Some(a),
KeyScope::Shared => None,
},
key: key.to_string(),
};
let live_id = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
let mut out = Vec::new();
let mut cursor = live_id;
while let Some(id_u) = cursor {
let idx = self.index.get(id_u, &self.catalog).ok_or_else(|| {
MnemoError::Invalid("history: superseded chain references missing id".into())
})?;
let entry = self.catalog[idx].clone();
if entry.deleted {
break;
}
let m = self.read_memory(&entry)?;
let next = m
.metadata
.get(MNEMO_META_SUPERSEDES)
.and_then(|v| v.as_str())
.and_then(|s| Ulid::from_string(s).ok())
.map(|u| u.0);
out.push(m);
cursor = next;
}
Ok(out)
}
pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let plan = self.plan_ingest(thread_id, turns)?;
if plan.pending.is_empty() {
return Ok(plan.report);
}
let vectors = {
let embedder = self.embedder.as_ref().ok_or_else(|| {
MnemoError::Invalid(
"ingest_thread: no embedder registered on this handle; call set_embedder first"
.into(),
)
})?;
embedder.embed(&plan.texts())?
};
self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
self.apply_ingest(thread_id, plan, vectors, None)
}
fn check_vectors(&self, ctx: &str, vectors: &[Vec<f32>], expected: usize) -> Result<()> {
if vectors.len() != expected {
return Err(MnemoError::Invalid(format!(
"{ctx}: embedder returned {} vectors for {expected} texts",
vectors.len()
)));
}
for v in vectors {
self.check_vector(ctx, v)?;
}
Ok(())
}
fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
if v.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: v.len(),
});
}
if embed::is_zero_vector(v) {
return Err(MnemoError::Invalid(format!(
"{ctx}: all-zero vector refused"
)));
}
if v.iter().any(|x| !x.is_finite()) {
return Err(MnemoError::Invalid(format!(
"{ctx}: vector contains a non-finite value"
)));
}
Ok(())
}
fn plan_ingest(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestPlan> {
if thread_id.is_empty() {
return Err(MnemoError::Invalid(
"ingest_thread: 'thread_id' must not be empty".into(),
));
}
let chunk_bytes = if self.is_v9() {
V9_TURN_CHUNK_BYTES
} else {
THREAD_TURN_CHUNK_BYTES
};
let mut report = IngestReport::default();
if turns.is_empty() {
return Ok(IngestPlan {
pending: Vec::new(),
next_seq: 0,
report,
});
}
self.ensure_key_caches()?;
self.warm_thread(thread_id)?;
let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
let existing_turn_ids: std::collections::HashSet<&str> = cached
.map(|tc| tc.turns.keys().map(String::as_str).collect())
.unwrap_or_default();
let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
let mut pending: Vec<PendingTurn> = Vec::new();
let now = memory::now_secs();
for t in turns {
if t.turn_id.is_empty() {
return Err(MnemoError::Invalid(
"ingest_thread: turn_id must not be empty".into(),
));
}
if existing_turn_ids.contains(t.turn_id.as_str()) {
report.skipped_existing_turns += 1;
continue;
}
let content_bytes = t.content.len();
let is_tool = t.role == "tool";
let (parts, truncation) =
if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
report.truncated_turns += 1;
let hash = {
use sha2::{Digest, Sha256};
let h = Sha256::digest(t.content.as_bytes());
hex_encode(&h)
};
let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
(vec![truncated], Some((hash, content_bytes as u64)))
} else if content_bytes > chunk_bytes {
report.chunked_turns += 1;
(chunk_utf8(&t.content, chunk_bytes), None)
} else {
(vec![t.content.clone()], None)
};
pending.push(PendingTurn {
turn_id: t.turn_id.clone(),
role: t.role.clone(),
created_at: t.timestamp.unwrap_or(now),
parts,
truncation,
});
}
Ok(IngestPlan {
pending,
next_seq,
report,
})
}
fn apply_ingest(
&mut self,
thread_id: &str,
plan: IngestPlan,
vectors: Vec<Vec<f32>>,
ttl: Option<i64>,
) -> Result<IngestReport> {
let IngestPlan {
pending,
mut next_seq,
mut report,
} = plan;
let mut vectors = vectors.into_iter();
for p in &pending {
let this_seq = next_seq;
next_seq += 1;
for (part_idx, part_text) in p.parts.iter().enumerate() {
let vec = vectors.next().ok_or_else(|| {
MnemoError::Invalid("ingest_thread: ran out of vectors".into())
})?;
let mut m =
Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
if let Some(t) = ttl {
m = m.with_ttl(t);
}
m.created_at = p.created_at;
m.accessed_at = p.created_at;
m.metadata.insert(
MNEMO_META_KIND.into(),
Value::String(MNEMO_META_KIND_TURN.into()),
);
m.metadata.insert(
MNEMO_META_THREAD.into(),
Value::String(thread_id.to_string()),
);
m.metadata
.insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
m.metadata
.insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
m.metadata
.insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
if p.parts.len() > 1 {
m.metadata.insert(
MNEMO_META_PART.into(),
Value::Number((part_idx as u64).into()),
);
}
if let Some((sha, len)) = &p.truncation {
m.metadata
.insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
m.metadata
.insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
}
self.put(m)?;
report.added_records += 1;
}
}
Ok(report)
}
pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
let mut out: Vec<Memory> = Vec::new();
for e in &entries {
let m = match decoded.remove(&e.id) {
Some(m) => self.finish_memory(e, m)?,
None => self.read_memory(e)?,
};
if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
continue;
}
if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
!= Some(MNEMO_META_KIND_TURN)
{
continue;
}
out.push(m);
}
out.sort_by(|a, b| {
let seq_a = a
.metadata
.get(MNEMO_META_SEQ)
.and_then(|v| v.as_u64())
.unwrap_or(0);
let seq_b = b
.metadata
.get(MNEMO_META_SEQ)
.and_then(|v| v.as_u64())
.unwrap_or(0);
let part_a = a
.metadata
.get(MNEMO_META_PART)
.and_then(|v| v.as_u64())
.unwrap_or(0);
let part_b = b
.metadata
.get(MNEMO_META_PART)
.and_then(|v| v.as_u64())
.unwrap_or(0);
seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
});
Ok(out)
}
pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
let records = self.thread_export(thread_id)?;
let mut out: Vec<ExportedTurn> = Vec::new();
let mut last_seq: Option<u64> = None;
for m in records {
let meta_str = |k: &str| {
m.metadata
.get(k)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string()
};
let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
if seq.is_some() && seq == last_seq {
if let Some(prev) = out.last_mut() {
prev.content.push_str(&m.content);
continue;
}
}
last_seq = seq;
out.push(ExportedTurn {
turn_id: meta_str(MNEMO_META_TURN_ID),
role: meta_str(MNEMO_META_ROLE),
timestamp: m.created_at,
sha256: m
.metadata
.get(MNEMO_META_SHA256)
.and_then(|v| v.as_str())
.map(String::from),
original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
content: m.content,
});
}
Ok(out)
}
pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let entries = self.thread_entries(thread_id)?;
let mut victims: Vec<Ulid> = Vec::new();
for e in &entries {
let m = self.read_memory(e)?;
if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
continue;
}
if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
!= Some(MNEMO_META_KIND_TURN)
{
continue;
}
victims.push(m.id);
}
let n = victims.len();
for id in &victims {
self.delete(id)?;
}
Ok(n)
}
pub fn essence_add(
&mut self,
scope: EssenceScope,
section: EssenceSection,
text: &str,
refs: Option<Vec<String>>,
) -> Result<Ulid> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
Self::check_essence_add(section, text)?;
self.essence_add_with(scope, section, text, refs, None)
}
fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
if section == EssenceSection::Timeline {
return Err(MnemoError::Invalid(
"essence_add: use essence_append_timeline for the timeline section".into(),
));
}
if text.is_empty() {
return Err(MnemoError::Invalid(
"essence_add: text must not be empty".into(),
));
}
if text.len() > section.item_max_bytes() {
return Err(MnemoError::Invalid(format!(
"essence_add: '{}' items must be <= {} bytes; got {}",
section.as_str(),
section.item_max_bytes(),
text.len()
)));
}
Ok(())
}
fn essence_add_with(
&mut self,
scope: EssenceScope,
section: EssenceSection,
text: &str,
refs: Option<Vec<String>>,
vector: Option<Vec<f32>>,
) -> Result<Ulid> {
let now = memory::now_secs();
let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
self.put(mem)
}
pub fn essence_append_timeline(
&mut self,
scope: EssenceScope,
text: &str,
refs: Option<Vec<String>>,
) -> Result<Ulid> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
Self::check_essence_append(text)?;
self.essence_append_with(scope, text, refs, None)
}
fn check_essence_append(text: &str) -> Result<()> {
if text.is_empty() {
return Err(MnemoError::Invalid(
"essence_append_timeline: text must not be empty".into(),
));
}
if text.len() > EssenceSection::Timeline.item_max_bytes() {
return Err(MnemoError::Invalid(format!(
"essence_append_timeline: entries must be <= {} bytes; got {}",
EssenceSection::Timeline.item_max_bytes(),
text.len()
)));
}
Ok(())
}
fn essence_append_with(
&mut self,
scope: EssenceScope,
text: &str,
refs: Option<Vec<String>>,
vector: Option<Vec<f32>>,
) -> Result<Ulid> {
let mem =
self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
self.put(mem)
}
pub fn essence_update(
&mut self,
id: &Ulid,
text: &str,
refs: Option<Vec<String>>,
) -> Result<WriteOutcome> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let existing = self.load_essence_record(id)?;
let section = self.section_of(&existing)?;
if section == EssenceSection::Timeline {
return Err(MnemoError::Invalid(
"essence_update: timeline entries are append-only; append a new one instead".into(),
));
}
if text.is_empty() {
return Err(MnemoError::Invalid(
"essence_update: text must not be empty".into(),
));
}
if text.len() > section.item_max_bytes() {
return Err(MnemoError::Invalid(format!(
"essence_update: '{}' items must be <= {} bytes; got {}",
section.as_str(),
section.item_max_bytes(),
text.len()
)));
}
let mut patch = MemoryPatch {
content: Some(text.to_string()),
..Default::default()
};
patch.vector = Some(existing.vector.clone());
if let Some(r) = refs {
let mut meta = Map::new();
meta.insert(
MNEMO_META_REFS.into(),
Value::Array(r.into_iter().map(Value::String).collect()),
);
patch.metadata = Some(meta);
}
self.update(id, patch)
}
pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
self.essence_transition_status(id, essence_status::RESOLVED)
}
pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
self.delete(id)
}
pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
let max_bytes = if max_bytes == 0 {
ESSENCE_DEFAULT_RENDER_BYTES
} else {
max_bytes
};
let entries = self.essence_entries()?;
let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
std::collections::BTreeMap::new();
for section in EssenceSection::render_order() {
by_section.insert(section.as_str(), Vec::new());
}
for e in &entries {
let m = self.read_memory(e)?;
if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
!= Some(MNEMO_META_KIND_ESSENCE)
{
continue;
}
let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
let matches_scope = match (&scope, rec_thread) {
(EssenceScope::Project, None) => true,
(EssenceScope::Thread(_), None) => true, (EssenceScope::Thread(t), Some(rt)) => rt == t,
(EssenceScope::Project, Some(_)) => false,
};
if !matches_scope {
continue;
}
let section_str = m
.metadata
.get(MNEMO_META_SECTION)
.and_then(|v| v.as_str())
.unwrap_or("timeline");
by_section
.entry(
EssenceSection::parse(section_str)
.map(|s| s.as_str())
.unwrap_or("timeline"),
)
.or_default()
.push(m);
}
for (name, items) in by_section.iter_mut() {
let newest_first = matches!(*name, "decisions" | "timeline");
items.sort_by(|a, b| {
let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
match (a_resolved, b_resolved) {
(false, true) => std::cmp::Ordering::Less,
(true, false) => std::cmp::Ordering::Greater,
_ => {
if newest_first {
b.created_at.cmp(&a.created_at)
} else {
a.created_at.cmp(&b.created_at)
}
}
}
});
}
let header_reserve = 256usize; let body_budget = max_bytes.saturating_sub(header_reserve);
let per_section_budget = |section: EssenceSection| -> usize {
match section {
EssenceSection::Intent => body_budget / 8,
EssenceSection::Conventions => body_budget / 6,
EssenceSection::Decisions => body_budget / 4,
EssenceSection::State => body_budget / 8,
EssenceSection::Open => body_budget / 8,
EssenceSection::Next => body_budget / 12,
EssenceSection::Lessons => body_budget / 12,
EssenceSection::Timeline => body_budget / 4,
}
};
let mut out = String::with_capacity(max_bytes);
let scope_label = match &scope {
EssenceScope::Thread(t) => format!("thread `{t}`"),
EssenceScope::Project => "project".to_string(),
};
out.push_str(&format!("# essence — {scope_label}\n\n"));
if let EssenceScope::Thread(new_id) = &scope {
if let Some(prev_id) = self.predecessor_thread(new_id)? {
let continuation = self
.render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
out.push_str(&continuation);
}
}
}
for section in EssenceSection::render_order() {
let items = match by_section.get(section.as_str()) {
Some(v) if !v.is_empty() => v,
_ => continue,
};
let budget = per_section_budget(*section);
let section_header = format!("## {}\n", section.as_str());
if out.len() + section_header.len() > max_bytes {
break;
}
out.push_str(§ion_header);
let mut used = 0usize;
let mut rendered = 0usize;
for item in items {
let line = render_essence_item(*section, item);
if used + line.len() > budget {
break;
}
if out.len() + line.len() > max_bytes {
break;
}
out.push_str(&line);
used += line.len();
rendered += 1;
}
if *section == EssenceSection::Timeline && rendered < items.len() {
let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
if out.len() + fold.len() <= max_bytes {
out.push_str(&fold);
}
}
if out.len() < max_bytes {
out.push('\n');
}
}
let pointer = match &scope {
EssenceScope::Thread(t) => format!(
"> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
),
EssenceScope::Project => String::from(
"> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
),
};
if out.len() + pointer.len() <= max_bytes {
out.push_str(&pointer);
}
Ok(out)
}
fn build_essence_record(
&self,
scope: EssenceScope,
section: EssenceSection,
text: &str,
refs: Option<Vec<String>>,
status_at: Option<i64>,
vector: Option<Vec<f32>>,
) -> Result<Memory> {
let v = match (vector, self.embedder.as_ref()) {
(Some(v), _) => v,
(None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
MnemoError::Invalid("essence: embedder returned no vector".into())
})?,
(None, None) => self.sentinel_vector(),
};
let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
m.metadata.insert(
MNEMO_META_KIND.into(),
Value::String(MNEMO_META_KIND_ESSENCE.into()),
);
m.metadata.insert(
MNEMO_META_SECTION.into(),
Value::String(section.as_str().into()),
);
if let EssenceScope::Thread(t) = &scope {
m.metadata
.insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
}
if let Some(r) = refs {
m.metadata.insert(
MNEMO_META_REFS.into(),
Value::Array(r.into_iter().map(Value::String).collect()),
);
}
if let Some(at) = status_at {
m.metadata.insert(
MNEMO_META_STATUS.into(),
Value::String(essence_status::OPEN.into()),
);
m.metadata.insert(
MNEMO_META_STATUS_HISTORY.into(),
Value::Array(vec![serde_json::json!({
"status": essence_status::OPEN,
"at": at,
})]),
);
}
Ok(m)
}
fn sentinel_vector(&self) -> Vec<f32> {
let dims = self.dimensions.max(1);
vec![1.0 / (dims as f32).sqrt(); dims]
}
fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
let m = self.get(id)?;
if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
{
return Err(MnemoError::Invalid(format!(
"essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
)));
}
Ok(m)
}
fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
let s = m
.metadata
.get(MNEMO_META_SECTION)
.and_then(|v| v.as_str())
.ok_or_else(|| {
MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
})?;
EssenceSection::parse(s)
.ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
}
fn essence_transition_status(&mut self, id: &Ulid, to: &str) -> Result<WriteOutcome> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let existing = self.load_essence_record(id)?;
let section = self.section_of(&existing)?;
if !section.has_status_lifecycle() {
return Err(MnemoError::Invalid(format!(
"essence_transition: '{}' section has no status lifecycle",
section.as_str()
)));
}
let from = existing
.metadata
.get(MNEMO_META_STATUS)
.and_then(|v| v.as_str())
.unwrap_or(essence_status::OPEN);
if from == to {
return Ok(WriteOutcome::Unchanged);
}
let allowed = ESSENCE_TRANSITIONS
.iter()
.any(|(f, t)| *f == from && *t == to);
if !allowed {
return Err(MnemoError::Invalid(format!(
"essence_transition: '{from}' → '{to}' is not a valid transition for essence"
)));
}
let mut new = existing.clone();
new.metadata
.insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
let now = memory::now_secs();
let history_entry = serde_json::json!({ "status": to, "at": now });
let history = new
.metadata
.entry(MNEMO_META_STATUS_HISTORY.to_string())
.or_insert_with(|| Value::Array(Vec::new()));
if let Value::Array(arr) = history {
arr.push(history_entry);
}
new.metadata
.insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
new.id = existing.id;
new.created_at = existing.created_at;
self.put(new)?;
Ok(WriteOutcome::Updated)
}
pub fn thread_start(&mut self, new_thread_id: &str, from_thread_id: &str) -> Result<Ulid> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if new_thread_id.is_empty() || from_thread_id.is_empty() {
return Err(MnemoError::Invalid(
"thread_start: 'new_thread_id' and 'from_thread_id' must not be empty".into(),
));
}
if new_thread_id == from_thread_id {
return Err(MnemoError::Invalid(
"thread_start: 'new_thread_id' must differ from 'from_thread_id'".into(),
));
}
let v = self.sentinel_vector();
let content = format!("thread link: {new_thread_id} continues from {from_thread_id}");
let mut m = Memory::new(content, MemoryType::Semantic, v).with_importance(0.1);
m.metadata.insert(
MNEMO_META_KIND.into(),
Value::String(MNEMO_META_KIND_THREAD_LINK.into()),
);
m.metadata.insert(
MNEMO_META_THREAD.into(),
Value::String(new_thread_id.to_string()),
);
m.metadata.insert(
MNEMO_META_CONTINUES.into(),
Value::String(from_thread_id.to_string()),
);
let id = self.put(m)?;
Ok(id)
}
fn render_continuation_prelude(
&mut self,
prev_thread: &str,
max_bytes: usize,
) -> Result<String> {
if max_bytes == 0 {
return Ok(String::new());
}
let entries = self.essence_entries()?;
let mut open_items: Vec<Memory> = Vec::new();
let mut next_items: Vec<Memory> = Vec::new();
let mut timeline_items: Vec<Memory> = Vec::new();
for e in &entries {
let m = self.read_memory(e)?;
if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
!= Some(MNEMO_META_KIND_ESSENCE)
{
continue;
}
if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(prev_thread) {
continue;
}
let section = match m
.metadata
.get(MNEMO_META_SECTION)
.and_then(|v| v.as_str())
.and_then(EssenceSection::parse)
{
Some(s) => s,
None => continue,
};
let status = m
.metadata
.get(MNEMO_META_STATUS)
.and_then(|v| v.as_str())
.unwrap_or(essence_status::OPEN);
match section {
EssenceSection::Open if status == essence_status::OPEN => {
open_items.push(m);
}
EssenceSection::Next if status == essence_status::OPEN => {
next_items.push(m);
}
EssenceSection::Timeline => timeline_items.push(m),
_ => {}
}
}
timeline_items.sort_by_key(|m| std::cmp::Reverse(m.created_at));
timeline_items.truncate(CONTINUATION_TIMELINE_ENTRIES);
timeline_items.reverse();
let mut out = String::new();
let header = format!("## continued from `{prev_thread}`\n");
if header.len() > max_bytes {
return Ok(String::new());
}
out.push_str(&header);
if !open_items.is_empty() {
let sub = "### open\n";
if out.len() + sub.len() <= max_bytes {
out.push_str(sub);
}
for m in &open_items {
let line = format!("- {}\n", m.content);
if out.len() + line.len() > max_bytes {
break;
}
out.push_str(&line);
}
}
if !next_items.is_empty() {
let sub = "### next\n";
if out.len() + sub.len() <= max_bytes {
out.push_str(sub);
}
for m in &next_items {
let line = format!("- {}\n", m.content);
if out.len() + line.len() > max_bytes {
break;
}
out.push_str(&line);
}
}
if !timeline_items.is_empty() {
let sub = "### recent milestones\n";
if out.len() + sub.len() <= max_bytes {
out.push_str(sub);
}
for m in &timeline_items {
let line = format!("- {}\n", m.content);
if out.len() + line.len() > max_bytes {
break;
}
out.push_str(&line);
}
}
if out.len() < max_bytes {
out.push('\n');
}
Ok(out)
}
fn predecessor_thread(&mut self, thread_id: &str) -> Result<Option<String>> {
self.ensure_key_caches()?;
Ok(self
.layers
.as_ref()
.and_then(|l| l.links.get(thread_id))
.map(|(from, _)| from.clone()))
}
pub fn commit_deltas(&mut self, batch: CommitBatch) -> Result<CommitReport> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let CommitBatch {
mut facts,
essence_ops,
turns,
thread_id,
turn_policy,
} = batch;
let has_embedder = self.embedder.is_some();
let in_ctx = |ctx: &str, e: MnemoError| MnemoError::Invalid(format!("{ctx}: {e}"));
for (i, f) in facts.iter().enumerate() {
if let Some(k) = &f.key {
if k.is_empty() {
return Err(MnemoError::Invalid(format!(
"commit_deltas: facts[{i}] key is empty"
)));
}
}
if f.memory.vector.is_empty() {
if f.memory.content.is_empty() {
return Err(MnemoError::Invalid(format!(
"commit_deltas: facts[{i}] has neither a vector nor content to embed"
)));
}
if !has_embedder {
return Err(MnemoError::Invalid(format!(
"commit_deltas: facts[{i}] has no vector and no embedder is registered \
on this handle; pass a vector or call set_embedder first"
)));
}
}
}
let mut removed_in_batch: std::collections::HashSet<u128> =
std::collections::HashSet::new();
for (i, op) in essence_ops.iter().enumerate() {
let ctx = format!("commit_deltas: essence_ops[{i}]");
match op.kind {
EssenceOpKind::Add => {
if op.scope.is_none() {
return Err(MnemoError::Invalid(format!("{ctx} Add missing 'scope'")));
}
let section = op.section.ok_or_else(|| {
MnemoError::Invalid(format!("{ctx} Add missing 'section'"))
})?;
let text = op
.text
.as_deref()
.ok_or_else(|| MnemoError::Invalid(format!("{ctx} Add missing 'text'")))?;
Self::check_essence_add(section, text).map_err(|e| in_ctx(&ctx, e))?;
}
EssenceOpKind::AppendTimeline => {
if op.scope.is_none() {
return Err(MnemoError::Invalid(format!(
"{ctx} AppendTimeline missing 'scope'"
)));
}
let text = op.text.as_deref().ok_or_else(|| {
MnemoError::Invalid(format!("{ctx} AppendTimeline missing 'text'"))
})?;
Self::check_essence_append(text).map_err(|e| in_ctx(&ctx, e))?;
}
EssenceOpKind::Update | EssenceOpKind::Resolve | EssenceOpKind::Remove => {
let id = op
.id
.ok_or_else(|| MnemoError::Invalid(format!("{ctx} needs 'id'")))?;
if removed_in_batch.contains(&id.0) {
return Err(MnemoError::Invalid(format!(
"{ctx}: record {id} is removed earlier in this batch"
)));
}
let existing = self.load_essence_record(&id).map_err(|e| in_ctx(&ctx, e))?;
let section = self.section_of(&existing).map_err(|e| in_ctx(&ctx, e))?;
match op.kind {
EssenceOpKind::Update => {
let text = op.text.as_deref().ok_or_else(|| {
MnemoError::Invalid(format!("{ctx} Update needs 'text'"))
})?;
if section == EssenceSection::Timeline {
return Err(MnemoError::Invalid(format!(
"{ctx}: timeline entries are append-only"
)));
}
if text.is_empty() || text.len() > section.item_max_bytes() {
return Err(MnemoError::Invalid(format!(
"{ctx}: '{}' items must be 1..={} bytes; got {}",
section.as_str(),
section.item_max_bytes(),
text.len()
)));
}
}
EssenceOpKind::Resolve => {
if !section.has_status_lifecycle() {
return Err(MnemoError::Invalid(format!(
"{ctx}: '{}' section has no status lifecycle",
section.as_str()
)));
}
}
_ => {
removed_in_batch.insert(id.0);
}
}
}
}
}
if !turns.is_empty() && thread_id.is_none() {
return Err(MnemoError::Invalid(
"commit_deltas: 'turns' non-empty but 'thread_id' missing".into(),
));
}
let archive_turns = !turns.is_empty() && !matches!(turn_policy, TurnPolicy::Drop);
if archive_turns && !has_embedder {
return Err(MnemoError::Invalid(
"commit_deltas: archiving 'turns' needs an embedder registered on this handle; \
call set_embedder first or use turn_policy Drop"
.into(),
));
}
let plan = match (&thread_id, archive_turns) {
(Some(t), true) => Some(self.plan_ingest(t, &turns)?),
_ => None,
};
let embeds_essence = |op: &EssenceOp| {
has_embedder && matches!(op.kind, EssenceOpKind::Add | EssenceOpKind::AppendTimeline)
};
let (vectors, n_texts) = {
let mut texts: Vec<&str> = Vec::new();
for f in &facts {
if f.memory.vector.is_empty() {
texts.push(f.memory.content.as_str());
}
}
for op in essence_ops.iter().filter(|op| embeds_essence(op)) {
texts.push(op.text.as_deref().expect("checked in step 1"));
}
if let Some(p) = &plan {
texts.extend(p.texts());
}
let vectors = match (texts.is_empty(), self.embedder.as_ref()) {
(true, _) => Vec::new(),
(false, Some(e)) => e.embed(&texts)?,
(false, None) => unreachable!("texts need an embedder; checked in step 1"),
};
(vectors, texts.len())
};
self.check_vectors("commit_deltas", &vectors, n_texts)?;
for (i, f) in facts.iter().enumerate() {
let v = &f.memory.vector;
if v.is_empty() {
continue;
}
if v.len() != self.dimensions {
return Err(MnemoError::Invalid(format!(
"commit_deltas: facts[{i}] vector has {} dims; this file is {}",
v.len(),
self.dimensions
)));
}
let ctx = format!("commit_deltas: facts[{i}]");
self.check_vector(&ctx, v)?;
}
let mut vectors = vectors.into_iter();
for f in facts.iter_mut() {
if f.memory.vector.is_empty() {
f.memory.vector = vectors.next().expect("one vector per text");
}
}
let essence_vectors: Vec<Option<Vec<f32>>> = essence_ops
.iter()
.map(|op| embeds_essence(op).then(|| vectors.next().expect("one vector per text")))
.collect();
let turn_vectors: Vec<Vec<f32>> = vectors.collect();
let mut report = CommitReport::default();
for f in facts {
match f.key {
Some(k) => {
let (_id, outcome) = self.remember_keyed(f.memory, &k)?;
match outcome {
WriteOutcome::Added => report.facts_added += 1,
WriteOutcome::Superseded | WriteOutcome::Updated => {
report.facts_superseded += 1
}
WriteOutcome::Unchanged => report.facts_unchanged += 1,
}
}
None => {
self.remember(f.memory)?;
report.facts_added += 1;
}
}
}
for (op, vector) in essence_ops.into_iter().zip(essence_vectors) {
let outcome = match op.kind {
EssenceOpKind::Add => {
self.essence_add_with(
op.scope.expect("checked in step 1"),
op.section.expect("checked in step 1"),
op.text.as_deref().expect("checked in step 1"),
op.refs,
vector,
)?;
WriteOutcome::Added
}
EssenceOpKind::AppendTimeline => {
self.essence_append_with(
op.scope.expect("checked in step 1"),
op.text.as_deref().expect("checked in step 1"),
op.refs,
vector,
)?;
WriteOutcome::Added
}
EssenceOpKind::Update => self.essence_update(
op.id.as_ref().expect("checked in step 1"),
op.text.as_deref().expect("checked in step 1"),
op.refs,
)?,
EssenceOpKind::Resolve => {
self.essence_resolve(op.id.as_ref().expect("checked in step 1"))?
}
EssenceOpKind::Remove => {
self.essence_remove(op.id.as_ref().expect("checked in step 1"))?;
WriteOutcome::Updated
}
};
match outcome {
WriteOutcome::Unchanged => report.essence_ops_unchanged += 1,
_ => report.essence_ops_applied += 1,
}
}
if let (Some(plan), Some(t)) = (plan, thread_id.as_deref()) {
let ttl = match turn_policy {
TurnPolicy::ExpireAfter(secs) => Some(secs),
_ => None,
};
let r = self.apply_ingest(t, plan, turn_vectors, ttl)?;
report.turns_added += r.added_records;
report.turns_skipped += r.skipped_existing_turns;
report.turns_truncated += r.truncated_turns;
report.turns_chunked += r.chunked_turns;
}
self.flush()?;
Ok(report)
}
#[doc(hidden)]
pub fn __format_version(&self) -> u16 {
self.header.version
}
#[doc(hidden)]
pub fn __free_list(&self) -> (Vec<(u64, u64, u64)>, usize) {
(
self.free
.to_runs()
.into_iter()
.map(|r| (r.freed_at, r.page, r.count))
.collect(),
self.pending_free.len(),
)
}
#[doc(hidden)]
pub fn __record_location(&self, id: &Ulid) -> Option<(u64, u32, u64)> {
let idx = self.index.get(id.0, &self.catalog)?;
let e = &self.catalog[idx];
(!e.deleted).then_some((e.start_page, e.page_count, e.gen))
}
#[doc(hidden)]
pub fn __cold_thread_buckets(&self) -> Option<usize> {
self.layers.as_ref().map(|l| l.cold_turns.len())
}
#[doc(hidden)]
pub fn __record_storage(&mut self, id: &Ulid) -> Result<(u32, bool)> {
let idx = self
.index
.get(id.0, &self.catalog)
.ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
let e = self.catalog[idx].clone();
let bytes = self.read_record(&e)?;
Ok((e.len, bytes.starts_with(&ZSTD_MAGIC)))
}
#[doc(hidden)]
pub fn __vector_column(&self) -> (bool, usize) {
(self.col.resident, self.col.page_rows.len())
}
#[doc(hidden)]
pub fn __catalog_shape(&self) -> (usize, usize, u64) {
(
self.catalog.len(),
self.delta_chain.len(),
self.header.catalog_start,
)
}
#[doc(hidden)]
pub fn __index_shape(&self) -> (usize, u64) {
(self.index_chain.len(), self.header.index_start)
}
#[doc(hidden)]
pub fn __bm25(&mut self, query: &str) -> Result<Vec<(Ulid, f32)>> {
let scores = self.bm25_scores(query, 1.2, 0.75)?;
let mut out: Vec<(Ulid, f32)> = scores
.into_iter()
.map(|(row, s)| (Ulid(self.catalog[row].id), s))
.collect();
out.sort_by(|a, b| b.1.total_cmp(&a.1));
Ok(out)
}
#[doc(hidden)]
pub fn __set_index_layout_for_testing(&mut self, layout: u8) {
self.index_layout_override = Some(layout);
}
#[doc(hidden)]
pub fn __bm25_segments(&self) -> usize {
self.bm25.manifest.segments.len()
}
#[doc(hidden)]
pub fn __next_page(&self) -> u64 {
self.header.next_page
}
#[doc(hidden)]
pub fn __add_key_slot_for_testing(&mut self, label: &str) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if !self.is_v9() {
return Err(MnemoError::Invalid("key slots need a v9 file".into()));
}
let mut wrapped = vec![0u8; WRAPPED_DEK_LEN];
wrapped[..crypto::KEY_LEN].copy_from_slice(&crypto::random_dek()[..]);
self.key_slots.slots.push(KeySlotEntry {
label: label.to_string(),
role: 0,
m_cost: KdfParams::raw_key().m_cost,
t_cost: KdfParams::raw_key().t_cost,
p_cost: KdfParams::raw_key().p_cost,
salt: crypto::random_salt(),
dek_nonce: crypto::random_nonce(),
wrapped_dek: wrapped,
});
self.dirty_keys = true;
Ok(())
}
#[doc(hidden)]
pub fn __key_slot_labels(&self) -> Vec<String> {
self.key_slots
.slots
.iter()
.map(|s| s.label.clone())
.collect()
}
#[doc(hidden)]
pub fn __key_slot_run(&self) -> (u64, u64, u64) {
(
self.header.keys.start,
self.header.keys.pages,
self.header.keys.gen,
)
}
#[doc(hidden)]
pub fn __ann_shortlist(
&mut self,
query: &[f32],
n_probe: Option<usize>,
n_rerank: Option<usize>,
) -> Result<Vec<Ulid>> {
self.ensure_ann()?;
Ok(match &self.ann {
Some(ann) => ann
.query(query, n_probe, n_rerank)
.into_iter()
.map(Ulid)
.collect(),
None => Vec::new(),
})
}
#[doc(hidden)]
pub fn __budget_report(&self) -> Result<BudgetReport> {
let h = &self.header;
let ps = h.page_size as u64;
let mut r = BudgetReport {
file_bytes: std::fs::metadata(&self.path)?.len(),
page_size: ps,
header_bytes: ps,
catalog_run_bytes: h.catalog_pages * ps,
catalog_payload_bytes: h.catalog_len,
index_run_bytes: h.index_pages * ps,
index_payload_bytes: h.index_len,
index_start_page: h.index_start,
wal_bytes: h.wal_pages * ps,
snapshot_manifest_bytes: h.manifest_pages * ps,
cache_dir_run_bytes: h.cache_pages * ps,
..Default::default()
};
for e in &self.catalog {
let packed = e.v9.is_some_and(|r| r.rec_slot != REC_SPAN);
let pages = if packed { 0 } else { e.page_count as u64 * ps };
if e.deleted {
if e.v9.is_none() {
r.deleted_record_page_bytes += pages;
}
} else {
r.record_page_bytes += pages;
r.record_payload_bytes += e.len as u64;
r.live_records += 1;
}
}
r.record_page_bytes += self.recs.page_rows.len() as u64 * ps;
r.record_page_bytes += self.col.page_rows.len() as u64 * ps;
let accounted = r.header_bytes
+ r.record_page_bytes
+ r.deleted_record_page_bytes
+ r.catalog_run_bytes
+ r.index_run_bytes
+ r.wal_bytes
+ r.snapshot_manifest_bytes
+ r.cache_dir_run_bytes;
r.unreferenced_bytes = r.file_bytes.saturating_sub(accounted);
r.ram_catalog = (self.catalog.capacity() * std::mem::size_of::<CatalogEntry>()) as u64;
r.ram_id_map = self.index.heap_bytes() as u64;
if let Some(ann) = &self.ann {
let (c, cb, ids, codes, assign) = ann.heap_bytes();
r.ram_ann_centroids = c as u64;
r.ram_ann_codebook = cb as u64;
r.ram_ann_posting_ids = ids as u64;
r.ram_ann_posting_codes = codes as u64;
r.ram_ann_assignment = assign as u64;
}
r.ram_cache_dir =
(self.cache_dir.capacity() * std::mem::size_of::<CacheDirectoryEntry>()) as u64;
let key_index = self.key_index.as_ref().map_or(0, |k| {
crate::index::hash_map_bytes::<KeySlot, u128>(k.capacity())
+ k.keys()
.map(|s| s.key.capacity() + s.scope_owner.as_ref().map_or(0, |o| o.capacity()))
.sum::<usize>()
});
let superseded = self.superseded.as_ref().map_or(0, |s| {
crate::index::hash_map_bytes::<u128, ()>(s.capacity())
});
r.ram_key_caches = (key_index + superseded) as u64;
r.ram_vector_column = (self.col.data.capacity() * 2
+ self.col.norms.capacity() * 4
+ self.col.have.capacity()) as u64;
r.ram_total = r.ram_catalog
+ r.ram_id_map
+ r.ram_ann_centroids
+ r.ram_ann_codebook
+ r.ram_ann_posting_ids
+ r.ram_ann_posting_codes
+ r.ram_ann_assignment
+ r.ram_cache_dir
+ r.ram_key_caches
+ r.ram_vector_column;
Ok(r)
}
pub fn memories(&mut self) -> Result<Vec<Memory>> {
self.ensure_column()?;
let entries: Vec<CatalogEntry> = self
.catalog
.iter()
.filter(|e| !e.deleted)
.cloned()
.collect();
let mut out = Vec::with_capacity(entries.len());
for e in &entries {
out.push(self.read_memory(e)?);
}
Ok(out)
}
pub fn about(&mut self) -> Result<Vec<Memory>> {
self.ensure_key_caches()?;
let ids: Vec<u128> = self
.layers
.as_ref()
.map(|l| l.onboarding.iter().copied().collect())
.unwrap_or_default();
let mut out: Vec<Memory> = Vec::new();
for e in self.live_entries(ids) {
let m = self.read_memory(&e)?;
if !m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) && is_onboarding(&m) {
out.push(m);
}
}
out.sort_by(|a, b| {
let a_manifest = is_manifest(a);
let b_manifest = is_manifest(b);
b_manifest
.cmp(&a_manifest)
.then_with(|| b.importance.total_cmp(&a.importance))
.then_with(|| a.created_at.cmp(&b.created_at))
});
Ok(out)
}
pub fn about_essence(&mut self) -> Result<Option<String>> {
let entries = self.essence_entries()?;
let mut any = false;
for e in &entries {
let m = self.read_memory(e)?;
let is_project_essence = m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
== Some(MNEMO_META_KIND_ESSENCE)
&& !m.metadata.contains_key(MNEMO_META_THREAD);
if is_project_essence && status_of(&m) != Some(essence_status::RESOLVED) {
any = true;
break;
}
}
if !any {
return Ok(None);
}
self.essence(EssenceScope::Project, ESSENCE_DEFAULT_RENDER_BYTES)
.map(Some)
}
pub fn search(
&mut self,
query: &[f32],
top_k: usize,
metric: Metric,
) -> Result<Vec<(Memory, f32)>> {
if query.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: query.len(),
});
}
let now = memory::now_secs();
let entries: Vec<CatalogEntry> = self
.catalog
.iter()
.filter(|e| !e.deleted)
.cloned()
.collect();
let mut scored: Vec<(Memory, f32)> = Vec::new();
for e in &entries {
let m = self.read_memory(e)?;
if m.is_expired(now) {
continue;
}
let sim = memory::similarity(metric, query, &m.vector);
scored.push((m, sim));
}
scored.sort_by(|a, b| b.1.total_cmp(&a.1));
scored.truncate(top_k);
Ok(scored)
}
fn score_once(
&mut self,
req: &RecallRequest,
now: i64,
force_exact: bool,
) -> Result<Vec<RecallResult>> {
self.kw = None;
if let Some(text) = &req.keywords {
let scores = self.bm25_scores(text, req.bm25_k1, req.bm25_b)?;
let max = scores.values().copied().fold(0.0f32, f32::max);
if max > 0.0 {
self.kw = Some(
scores
.into_iter()
.map(|(row, s)| (self.catalog[row].id, s / max))
.collect(),
);
} else {
self.kw = Some(std::collections::HashMap::new());
}
}
if let Some(thread) = filter_thread_pin(&req.filter) {
let thread = thread.to_string();
let entries = self.thread_entries(&thread)?;
return self.score_entries(req, now, &entries);
}
if req.query.is_empty() {
let ids: Vec<u128> = self
.kw
.as_ref()
.map_or(Vec::new(), |k| k.keys().copied().collect());
let entries: Vec<CatalogEntry> = ids
.into_iter()
.filter_map(|id| self.index.get(id, &self.catalog))
.map(|i| self.catalog[i].clone())
.filter(|e| !e.deleted)
.collect();
return self.score_entries(req, now, &entries);
}
let force_exact = force_exact || filter_bypasses_ann(&req.filter);
let mut entries: Vec<CatalogEntry> = match (&self.ann, force_exact) {
(Some(ann), false) => {
let n_rerank = self.rerank_for(ann, req);
let ids = ann.query(&req.query, req.n_probe, n_rerank);
ids.iter()
.filter_map(|&id| self.index.get(id, &self.catalog))
.map(|i| self.catalog[i].clone())
.filter(|e| !e.deleted)
.collect()
}
_ => self
.catalog
.iter()
.filter(|e| !e.deleted)
.cloned()
.collect(),
};
if let Some(kw) = &self.kw {
if entries.len() < self.catalog.len() {
let have: std::collections::HashSet<u128> = entries.iter().map(|e| e.id).collect();
let extra: Vec<u128> = kw.keys().copied().filter(|id| !have.contains(id)).collect();
for id in extra {
if let Some(i) = self.index.get(id, &self.catalog) {
if !self.catalog[i].deleted {
entries.push(self.catalog[i].clone());
}
}
}
}
}
self.score_entries(req, now, &entries)
}
fn relevance(&self, req: &RecallRequest, id: u128, sim: f32) -> (f32, f32) {
let kw = self
.kw
.as_ref()
.and_then(|k| k.get(&id).copied())
.unwrap_or(0.0);
if req.query.is_empty() {
(kw, kw)
} else {
(sim + req.keyword_weight * kw, sim)
}
}
fn score_entries(
&mut self,
req: &RecallRequest,
now: i64,
entries: &[CatalogEntry],
) -> Result<Vec<RecallResult>> {
if self.is_v9() && req.filter.is_empty() {
return self.score_rows(req, now, entries);
}
let mut scored: Vec<RecallResult> = Vec::with_capacity(entries.len());
for e in entries {
if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
continue;
}
let m = self.read_memory(e)?;
if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
continue;
}
if m.is_expired(now) {
continue;
}
if let Some(types) = &req.memory_types {
if !types.contains(&m.memory_type) {
continue;
}
}
if let Some(agent) = &req.agent_id {
let visible = m.agent_id == *agent || m.scope == Scope::Shared;
if !visible {
continue;
}
}
if !req.filter.is_empty() && !req.filter.matches(&m) {
continue;
}
if !req.filter.mentions_mnemo_meta()
&& matches!(
m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
Some(MNEMO_META_KIND_TURN)
| Some(MNEMO_META_KIND_ESSENCE)
| Some(MNEMO_META_KIND_THREAD_LINK)
)
{
continue;
}
let sim = if req.query.is_empty() {
0.0
} else {
memory::similarity(req.metric, &req.query, &m.vector)
};
let (rel, sim) = self.relevance(req, e.id, sim);
let age_secs = (now - m.created_at) as f32;
let score = req
.weights
.score(rel, age_secs, m.importance, m.access_count);
scored.push(RecallResult {
memory: m,
score,
similarity: sim,
});
}
Ok(scored)
}
fn score_rows(
&mut self,
req: &RecallRequest,
now: i64,
entries: &[CatalogEntry],
) -> Result<Vec<RecallResult>> {
self.ensure_column()?;
let d = self.dimensions;
let q = &req.query;
let q_norm = q.iter().map(|x| x * x).sum::<f32>().sqrt();
let agent_h = req.agent_id.as_deref().map(|a| fnv1a32(&[a.as_bytes()]));
let types: Option<Vec<u8>> = req
.memory_types
.as_ref()
.map(|ts| ts.iter().map(|&t| memory_type_code(t)).collect());
let mut cand: Vec<(f32, f32, usize)> = Vec::with_capacity(entries.len());
for (pos, e) in entries.iter().enumerate() {
let Some(r) = e.v9 else { continue };
if r.flags & ROW_SUPERSEDED != 0 {
continue;
}
if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
continue;
}
if r.flags & ROW_HAS_TTL != 0 && now >= r.expires_at {
continue;
}
if let Some(ts) = &types {
if !ts.contains(&r.memory_type) {
continue;
}
}
if let Some(ah) = agent_h {
if r.flags & ROW_SHARED == 0 && r.agent_hash != ah {
continue;
}
}
if matches!(
r.kind,
ROW_KIND_TURN | ROW_KIND_ESSENCE | ROW_KIND_THREAD_LINK
) {
continue;
}
let Some(idx) = self.index.get(e.id, &self.catalog) else {
continue;
};
if !req.query.is_empty()
&& self.col.resident
&& idx < self.col.have.len()
&& !self.col.have[idx]
&& !self.col.hot.contains_key(&idx)
{
self.fill_column_page(idx)?;
}
let sim = if req.query.is_empty() {
0.0
} else if let Some((h, n)) = self.col.hot.get(&idx) {
veccol::similarity_f16(req.metric, q, q_norm, h, *n)
} else if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
veccol::similarity_f16(
req.metric,
q,
q_norm,
&self.col.data[idx * d..(idx + 1) * d],
self.col.norms[idx],
)
} else {
let (h, n) = self.column_entry(e)?;
veccol::similarity_f16(req.metric, q, q_norm, &h, n)
};
let (rel, sim) = self.relevance(req, e.id, sim);
let score = req.weights.score(
rel,
(now - r.created_at) as f32,
r.importance,
e.access_count,
);
cand.push((score, sim, pos));
}
cand.sort_by(|a, b| b.0.total_cmp(&a.0));
let mut out: Vec<RecallResult> = Vec::with_capacity(req.top_k.min(cand.len()));
for (score, sim, pos) in cand {
if out.len() >= req.top_k {
break;
}
let m = self.read_memory(&entries[pos])?;
if let Some(agent) = &req.agent_id {
if !(m.agent_id == *agent || m.scope == Scope::Shared) {
continue;
}
}
out.push(RecallResult {
memory: m,
score,
similarity: sim,
});
}
Ok(out)
}
pub fn recall(&mut self, req: &RecallRequest) -> Result<Vec<RecallResult>> {
Ok(self.recall_budgeted(req)?.hits)
}
pub fn recall_budgeted(&mut self, req: &RecallRequest) -> Result<RecallResponse> {
if req.keywords.is_some() && !self.is_v9() {
return Err(MnemoError::Invalid(
"keyword recall (keywords) needs a v9 file; run `mnemo compact` to upgrade".into(),
));
}
let keyword_only = req.query.is_empty() && req.keywords.is_some();
if !keyword_only && req.query.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: req.query.len(),
});
}
self.ensure_ann()?;
if self.read_only && req.track_access {
return Err(MnemoError::ReadOnly);
}
let live_count = self.catalog.iter().filter(|e| !e.deleted).count();
let live_non_superseded = live_count;
let now = memory::now_secs();
let mut scored = self.score_once(req, now, false)?;
let mut used_exact_scan = self.ann.is_none() || filter_bypasses_ann(&req.filter);
if !used_exact_scan && scored.len() < req.top_k && live_non_superseded > req.top_k {
if let Some(ann) = &self.ann {
let (default_probe, default_rerank) = ann.effective_defaults();
let base_probe = req.n_probe.unwrap_or(default_probe);
let base_rerank = self.rerank_for(ann, req).unwrap_or(default_rerank);
let widen_probe = base_probe.saturating_mul(2);
let widen_rerank = base_rerank.saturating_mul(2);
if widen_probe > base_probe || widen_rerank > base_rerank {
let widened = RecallRequest {
n_probe: Some(widen_probe),
n_rerank: Some(widen_rerank),
..req.clone()
};
scored = self.score_once(&widened, now, false)?;
}
if scored.len() < req.top_k {
scored = self.score_once(req, now, true)?;
used_exact_scan = true;
}
}
}
scored.sort_by(|a, b| b.score.total_cmp(&a.score));
scored.truncate(req.top_k);
if req.track_access {
for r in &mut scored {
if let Some(idx) = self.index.get(r.memory.id.0, &self.catalog) {
self.touch(idx);
let entry = &mut self.catalog[idx];
entry.accessed_at = now;
entry.access_count = entry.access_count.saturating_add(1);
r.memory.accessed_at = entry.accessed_at;
r.memory.access_count = entry.access_count;
}
}
if !scored.is_empty() {
self.dirty_catalog = true;
}
}
let truncated = if let Some(budget) = req.max_bytes {
let orig_len = scored.len();
let mut running = 0usize;
let mut cut_at: Option<usize> = None;
for (i, r) in scored.iter().enumerate() {
let w = r.memory.content.len();
if i == 0 {
running += w;
continue;
}
if running.saturating_add(w) > budget {
cut_at = Some(i);
break;
}
running += w;
}
if let Some(idx) = cut_at {
scored.truncate(idx);
scored.len() < orig_len
} else {
false
}
} else {
false
};
self.kw = None;
Ok(RecallResponse {
hits: scored,
truncated,
used_exact_scan,
})
}
pub fn build_index(&mut self) -> Result<IndexInfo> {
self.build_index_with(IndexConfig::default())
}
pub fn build_index_with(&mut self, cfg: IndexConfig) -> Result<IndexInfo> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let mems: Vec<Memory> = self.memories()?.into_iter().filter(is_ann_kind).collect();
if mems.is_empty() {
return Err(MnemoError::Invalid(
"cannot build an index over an empty database".into(),
));
}
let min_facts = std::env::var("MNEMO_DEV_ANN_MIN_FACTS")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or(V9_ANN_MIN_FACTS);
if self.is_v9() && mems.len() < min_facts {
if self.ann.is_some() || self.ann_unloaded {
self.ann = None;
self.ann_unloaded = false;
self.dirty_index = true;
self.index_persist.forced = true;
}
return Ok(IndexInfo {
vectors: mems.len(),
partitions: 0,
subspaces: 0,
n_probe: 0,
n_rerank: 0,
});
}
let items: Vec<(u128, &[f32])> =
mems.iter().map(|m| (m.id.0, m.vector.as_slice())).collect();
let mut idx = IvfPqIndex::build(self.dimensions, &items, cfg)?;
let info = idx.info();
if self.is_v9() {
idx.enable_log();
self.index_rebase = true;
}
self.ann = Some(idx);
self.ann_unloaded = false;
self.dirty_index = true;
self.index_persist.forced = true;
Ok(info)
}
pub fn rebuild_index(&mut self) -> Result<IndexInfo> {
self.build_index_with(IndexConfig::default())
}
pub fn drop_index(&mut self) {
if self.read_only {
return;
}
if self.ann.is_some() || self.ann_unloaded {
self.ann = None;
self.ann_unloaded = false;
self.dirty_index = true;
self.index_persist.forced = true;
}
}
fn reconcile_ann_with_catalog(&mut self, index_start: u64, index_gen: u64) -> Result<()> {
if self.ann.is_none() {
return Ok(());
}
let mut changed = 0usize;
let stale: Vec<u128> = match &self.ann {
Some(ann) => ann
.ids()
.filter(|&id| !matches!(self.index.get(id, &self.catalog), Some(i) if !self.catalog[i].deleted))
.collect(),
None => Vec::new(),
};
if let Some(ann) = &mut self.ann {
for id in &stale {
ann.remove(*id);
changed += 1;
}
}
let v9 = self.is_v9();
let pending: Vec<CatalogEntry> = self
.catalog
.iter()
.filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
.cloned()
.collect();
for e in &pending {
let m = self.read_memory(e)?;
if let Some(ann) = &mut self.ann {
if is_ann_kind(&m) {
ann.add(e.id, &m.vector);
} else {
ann.remove(e.id);
}
changed += 1;
}
}
if changed > 0 {
self.dirty_index = true;
self.index_persist.changes += changed;
}
Ok(())
}
pub fn has_index(&self) -> bool {
self.ann.is_some() || self.ann_unloaded
}
fn ensure_ann(&mut self) -> Result<()> {
if !self.ann_unloaded {
return Ok(());
}
self.ann_unloaded = false;
self.ann = load_index(
&mut self.pager,
self.header.index_start,
self.header.index_pages,
self.header.index_len,
self.header.index_gen,
self.dimensions,
)?;
if self.is_v9() {
if let Some(ann) = self.ann.as_mut() {
let (chain, n) = apply_index_chain(&mut self.pager, ann, self.header.index_delta)?;
self.index_chain = chain;
self.index_chain_ops = n;
ann.enable_log();
} else if self.header.index_pages > 0 {
self.index_chain = index_chain_runs(&mut self.pager, self.header.index_delta);
self.index_chain_ops = 0;
}
}
let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
self.reconcile_ann_with_catalog(index_start, index_gen)?;
let non_facts: Vec<u128> = match &self.layers {
Some(l) => l
.essence
.iter()
.copied()
.chain(l.threads.values().flat_map(|t| t.records.iter().copied()))
.chain(l.cold_turns.values().flatten().copied())
.collect(),
None => Vec::new(),
};
if let Some(ann) = &mut self.ann {
let mut pruned = 0usize;
for id in non_facts {
if ann.contains(id) {
ann.remove(id);
pruned += 1;
}
}
if pruned > 0 {
self.dirty_index = true;
self.index_persist.changes += pruned;
}
}
Ok(())
}
fn seal_run(
&mut self,
bytes: &[u8],
gen: u64,
frames: &mut Vec<wal::Frame>,
) -> Result<(u64, u32)> {
let pc = bytes.len().div_ceil(PAYLOAD).max(1);
let start = self.alloc_pages(pc as u64);
self.seal_run_at(start, pc as u64, bytes, gen, frames)?;
Ok((start, pc as u32))
}
fn seal_run_at(
&mut self,
start: u64,
pages: u64,
bytes: &[u8],
gen: u64,
frames: &mut Vec<wal::Frame>,
) -> Result<()> {
let direct = self.is_v9() && pages > V9_WAL_RUN_MAX_PAGES;
for i in 0..pages as usize {
let lo = (i * PAYLOAD).min(bytes.len());
let hi = ((i + 1) * PAYLOAD).min(bytes.len());
let page_no = start + i as u64;
let sealed = self.pager.seal_page(page_no, gen, &bytes[lo..hi])?;
if direct {
self.pager.write_sealed(page_no, &sealed)?;
} else {
frames.push((page_no, sealed.to_vec()));
}
}
if direct {
self.direct_writes = true;
}
Ok(())
}
fn ensure_wal_capacity(&mut self, frame_pages: usize) -> Result<()> {
let need = wal::txn_byte_len(frame_pages, PAGE_SIZE);
if need <= self.header.wal_pages * PAGE_SIZE as u64 {
return Ok(());
}
let req = need.div_ceil(PAGE_SIZE as u64);
let new_pages = (req + req / 2 + 4).max(DEFAULT_WAL_PAGES);
let old_wal_start = self.header.wal_start;
let old_wal_pages = self.header.wal_pages;
let new_start = self.header.next_page;
self.header.next_page += new_pages;
self.header.wal_start = new_start;
self.header.wal_pages = new_pages;
self.header.wal_seq += 1;
self.relocate_header_through_wal(old_wal_start)?;
self.release_pages(old_wal_start, old_wal_pages);
Ok(())
}
fn relocate_header_through_wal(&mut self, old_wal_start: u64) -> Result<()> {
self.header.write_counter = self.pager.write_counter;
let mut page = self.header.to_page();
self.header.apply_seal(&mut page, self.pager.dek())?;
let txn_id = self.header.wal_seq;
let frames = vec![(0u64, page.to_vec())];
wal::commit(
self.pager.file_mut(),
self.header.wal_start,
self.header.wal_pages,
txn_id,
&frames,
)?;
self.pager.write_raw(0, &page)?;
self.pager.sync()?;
if old_wal_start != self.header.wal_start {
wal::invalidate_region(self.pager.file_mut(), old_wal_start)?;
}
Ok(())
}
fn checkpoint(&mut self, frames: &[wal::Frame]) -> Result<()> {
for (page_no, bytes) in frames {
let mut img = [0u8; PAGE_SIZE];
img.copy_from_slice(bytes);
self.pager.write_sealed(*page_no, &img)?;
}
self.pager.sync()?;
Ok(())
}
pub fn flush(&mut self) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let fsync_before = crate::fsync_count();
if self.ann_unloaded && self.dirty_catalog {
let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
let v9 = self.is_v9();
let pending = self
.catalog
.iter()
.filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
.count();
if pending >= INDEX_PERSIST_EVERY_CHANGES {
self.ensure_ann()?;
}
}
let persist_index = if self.is_v9() {
self.plan_index_v9()
} else {
self.dirty_index
&& (self.index_persist.forced
|| self.ann.is_none()
|| self.index_persist.commits + 1 >= INDEX_PERSIST_EVERY_COMMITS
|| self.index_persist.changes >= INDEX_PERSIST_EVERY_CHANGES)
};
let free_work =
self.is_v9() && (self.free_dirty || !self.pending_free.is_empty() || self.dirty_keys);
if !self.dirty_catalog && !persist_index && !self.dirty_cache && !free_work {
self.fsyncs_last_flush = 0;
return Ok(());
}
let is_memory_or_index_flush = self.dirty_catalog || persist_index;
self.ensure_gen_lease()?;
self.settle_cold_turns()?;
self.pack_records()?;
self.pack_vectors()?;
let ctx = self.prepare_for_flush(persist_index)?;
self.pager.flush()?;
let txn_id = self.header.wal_seq + 1;
let gen = if self.is_v9() { txn_id } else { 0 };
let v9 = self.is_v9();
let mut superseded: Vec<(u64, u64)> = Vec::new();
let mut frames: Vec<wal::Frame> = Vec::new();
let mut new_chain: Option<(Vec<RunPtr>, usize)> = None;
if let Some(bytes) = &ctx.cat_bytes {
if ctx.cat_delta {
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
let ptr = RunPtr {
start,
pages: pages as u64,
len: bytes.len() as u64,
gen,
};
self.header.catalog_delta = ptr;
let mut chain = self.delta_chain.clone();
chain.push(ptr);
new_chain = Some((chain, self.delta_rows + self.dirty_rows.len()));
} else {
if v9 && self.header.catalog_pages > 0 {
superseded.push((self.header.catalog_start, self.header.catalog_pages));
}
if v9 {
for r in &self.delta_chain {
superseded.push((r.start, r.pages));
}
self.header.catalog_delta = RunPtr::default();
new_chain = Some((Vec::new(), 0));
}
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
self.header.catalog_start = start;
self.header.catalog_pages = pages as u64;
self.header.catalog_len = bytes.len() as u64;
self.header.catalog_gen = gen;
}
}
let mut new_bm25: Option<Bm25Segments> = None;
if let Some((bytes, replace_from)) = &ctx.bm25_seg {
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
let mut segs = self.bm25.manifest.segments.clone();
let mut seg_docs = self.bm25.manifest.seg_docs.clone();
seg_docs.resize(segs.len(), 0);
let docs = if let Some(from) = *replace_from {
for t in segs.drain(from..) {
superseded.push((t.0, t.1));
}
seg_docs.truncate(from);
self.bm25.merged_docs
} else {
self.bm25.pending.len() as u32
};
segs.push((start, pages as u64, bytes.len() as u64, gen));
seg_docs.push(docs);
let man = Bm25Manifest {
tokenizer: text::TOKENIZER_V1,
next_doc: self.bm25.manifest.next_doc,
segments: segs.clone(),
seg_docs: seg_docs.clone(),
};
let mbytes =
rmp_serde::to_vec(&man).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if !self.header.bm25.is_empty() {
superseded.push((self.header.bm25.start, self.header.bm25.pages));
}
let (ms, mp) = self.seal_run(&mbytes, gen, &mut frames)?;
self.header.bm25 = RunPtr {
start: ms,
pages: mp as u64,
len: mbytes.len() as u64,
gen,
};
new_bm25 = Some((segs, seg_docs));
}
let mut new_ichain: Option<(Vec<RunPtr>, usize)> = None;
if persist_index && ctx.idx_delta {
if let Some(bytes) = &ctx.idx_bytes {
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
let ptr = RunPtr {
start,
pages: pages as u64,
len: bytes.len() as u64,
gen,
};
self.header.index_delta = ptr;
let mut chain = self.index_chain.clone();
chain.push(ptr);
let n = self.index_delta_ops.as_ref().map_or(0, |o| o.len());
new_ichain = Some((chain, self.index_chain_ops + n));
}
} else if persist_index {
if v9 && self.header.index_pages > 0 {
superseded.push((self.header.index_start, self.header.index_pages));
}
if v9 {
for r in &self.index_chain {
superseded.push((r.start, r.pages));
}
self.header.index_delta = RunPtr::default();
new_ichain = Some((Vec::new(), 0));
}
match &ctx.idx_bytes {
Some(bytes) => {
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
self.header.index_start = start;
self.header.index_pages = pages as u64;
self.header.index_len = bytes.len() as u64;
self.header.index_gen = gen;
}
None => {
self.header.index_start = 0;
self.header.index_pages = 0;
self.header.index_len = 0;
self.header.index_gen = 0;
}
}
}
if self.dirty_cache {
if v9 && self.header.cache_pages > 0 {
superseded.push((self.header.cache_start, self.header.cache_pages));
}
match &ctx.cache_bytes {
Some(bytes) if !self.cache_dir.is_empty() => {
let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
self.header.cache_start = start;
self.header.cache_pages = pages as u64;
self.header.cache_len = bytes.len() as u64;
self.header.cache_gen = gen;
}
_ => {
self.header.cache_start = 0;
self.header.cache_pages = 0;
self.header.cache_len = 0;
self.header.cache_gen = 0;
}
}
}
let live = self.len() as u64;
let mut manifest = self.manifest.clone();
if is_memory_or_index_flush {
if self.max_snapshots > 0 && !v9 {
while manifest.len() >= self.max_snapshots {
manifest.remove(0);
}
}
manifest.push(Snapshot {
txn_id,
created_at: memory::now_secs(),
catalog_start: self.header.catalog_start,
catalog_pages: self.header.catalog_pages,
catalog_len: self.header.catalog_len,
index_start: self.header.index_start,
index_pages: self.header.index_pages,
index_len: self.header.index_len,
memory_count: live,
gens: v9.then_some(SnapshotGens {
catalog: self.header.catalog_gen,
index: self.header.index_gen,
delta: ptr_tuple(self.header.catalog_delta),
index_delta: ptr_tuple(self.header.index_delta),
bm25: ptr_tuple(self.header.bm25),
}),
});
if v9 {
let extra = self.pending_free.iter().map(|r| r.1).sum::<u64>()
+ superseded.iter().map(|r| r.1).sum::<u64>()
+ self.header.manifest_pages;
self.apply_retention(&mut manifest, extra);
}
let man_bytes =
rmp_serde::to_vec(&manifest).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if v9 && self.header.manifest_pages > 0 {
superseded.push((self.header.manifest_start, self.header.manifest_pages));
}
let (m_start, m_pages) = self.seal_run(&man_bytes, gen, &mut frames)?;
self.header.manifest_start = m_start;
self.header.manifest_pages = m_pages as u64;
self.header.manifest_len = man_bytes.len() as u64;
self.header.manifest_gen = gen;
}
if v9 && self.dirty_keys {
if !self.header.keys.is_empty() {
superseded.push((self.header.keys.start, self.header.keys.pages));
}
if self.key_slots.slots.is_empty() {
self.header.keys = RunPtr::default();
} else {
let bytes = rmp_serde::to_vec(&self.key_slots)
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
let (start, pages) = self.seal_run(&bytes, gen, &mut frames)?;
self.header.keys = RunPtr {
start,
pages: pages as u64,
len: bytes.len() as u64,
gen,
};
}
}
let mut shrink = false;
let staged_free =
if v9 && (self.free_dirty || !self.pending_free.is_empty() || !superseded.is_empty()) {
let mut staged = self.free.clone();
for &(p, c) in self.pending_free.iter().chain(superseded.iter()) {
staged.free(txn_id, p, c)?;
}
if !self.header.free.is_empty() {
staged.free(txn_id, self.header.free.start, self.header.free.pages)?;
}
let min_snap = self.min_snapshot_txn();
while let Some(p) =
staged.take_tail(self.header.next_page, self.header.wal_seq, min_snap)
{
self.header.next_page = p;
shrink = true;
}
let est = rmp_serde::to_vec(&staged.to_runs())
.map_err(|e| MnemoError::Serialize(e.to_string()))?
.len()
+ 64;
let pc = est.div_ceil(PAYLOAD).max(1) as u64;
let start = match staged.alloc(pc, self.header.wal_seq, self.min_snapshot_txn()) {
Some(p) => p,
None => {
let p = self.header.next_page;
self.header.next_page += pc;
p
}
};
let bytes = rmp_serde::to_vec(&staged.to_runs())
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
if bytes.len() as u64 > pc * PAYLOAD as u64 {
return Err(MnemoError::Invalid(
"free list outgrew its reserved run".into(),
));
}
self.seal_run_at(start, pc, &bytes, gen, &mut frames)?;
self.header.free = RunPtr {
start,
pages: pc,
len: bytes.len() as u64,
gen,
};
Some(staged)
} else {
None
};
self.header.vector_count = live;
self.header.write_counter = self.pager.write_counter;
self.header.wal_seq = txn_id;
let mut hpage = self.header.to_page();
self.header.apply_seal(&mut hpage, self.pager.dek())?;
frames.push((0, hpage.to_vec()));
if self.direct_writes {
self.pager.sync()?;
self.direct_writes = false;
}
let (wal_start, wal_pages) = (self.header.wal_start, self.header.wal_pages);
wal::commit(self.pager.file_mut(), wal_start, wal_pages, txn_id, &frames)?;
self.checkpoint(&frames)?;
if shrink {
let len = self.header.next_page * PAGE_SIZE as u64;
self.pager.file_mut().set_len(len)?;
}
self.manifest = manifest;
if let Some(staged) = staged_free {
self.free = staged;
}
self.pending_free.clear();
self.free_dirty = false;
self.dirty_keys = false;
if let Some((chain, rows)) = new_chain {
self.delta_chain = chain;
self.delta_rows = rows;
}
if let Some((chain, n)) = new_ichain {
self.index_chain = chain;
self.index_chain_ops = n;
}
if let Some((segs, seg_docs)) = new_bm25 {
let kept = self
.bm25
.manifest
.segments
.iter()
.zip(&segs)
.take_while(|(a, b)| a == b)
.count();
if let Some(mut dicts) = self.bm25.dicts.take() {
dicts.truncate(kept);
let mut ok = dicts.len() == kept;
for &t in &segs[kept..] {
match load_segment_dict(&mut self.pager, tuple_ptr(t)) {
Ok(d) if ok => dicts.push(d),
_ => ok = false,
}
}
if ok {
self.bm25.dicts = Some(dicts);
}
}
self.bm25.manifest.segments = segs;
self.bm25.manifest.seg_docs = seg_docs;
self.bm25.pending.clear();
}
if persist_index {
self.index_rebase = false;
self.index_delta_ops = None;
}
if ctx.cat_bytes.is_some() {
self.dirty_rows.clear();
self.catalog_rewrite = false;
}
self.dirty_catalog = false;
if persist_index {
self.dirty_index = false;
self.index_persist = IndexPersist::default();
} else if self.dirty_index {
self.index_persist.commits += 1;
}
self.dirty_cache = false;
self.cache_batch.reset();
self.fsyncs_last_flush =
(crate::fsync_count().wrapping_sub(fsync_before)).min(u32::MAX as u64) as u32;
Ok(())
}
fn prepare_for_flush(&mut self, persist_index: bool) -> Result<FlushPrelude> {
let cat_delta = self.dirty_catalog && self.catalog_as_delta();
let cat_bytes = if !self.dirty_catalog {
None
} else if cat_delta {
let rows: Vec<(u32, CatalogEntry)> = self
.dirty_rows
.iter()
.filter(|&&i| i < self.catalog.len())
.map(|&i| (i as u32, self.catalog[i].clone()))
.collect();
let d = CatalogDelta {
prev: ptr_tuple(self.header.catalog_delta),
rows,
};
Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
} else {
if self.is_v9() {
self.compact_tombstones();
}
Some(
rmp_serde::to_vec(&self.catalog)
.map_err(|e| MnemoError::Serialize(e.to_string()))?,
)
};
let idx_delta = persist_index && self.index_delta_ops.is_some();
let idx_bytes: Option<Vec<u8>> = if !persist_index {
None
} else if let Some(ops) = &self.index_delta_ops {
let d = IndexDeltaRef {
prev: ptr_tuple(self.header.index_delta),
ops,
};
Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
} else {
match &self.ann {
Some(ann) => {
let mut b =
rmp_serde::to_vec(ann).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if self.is_v9() {
b.insert(0, self.index_layout_override.unwrap_or(V9_INDEX_LAYOUT));
}
Some(b)
}
None => None,
}
};
let cache_bytes: Option<Vec<u8>> = if self.dirty_cache {
Some(
rmp_serde::to_vec(&self.cache_dir)
.map_err(|e| MnemoError::Serialize(e.to_string()))?,
)
} else {
None
};
let cat_pc = cat_bytes
.as_ref()
.map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
let idx_pc = idx_bytes
.as_ref()
.map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
let cache_pc = cache_bytes
.as_ref()
.map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
let per_snapshot = if self.is_v9() { 102 } else { 82 };
let man_upper = 9 + (self.manifest.len() + 1) * per_snapshot;
let man_pc_est = man_upper.div_ceil(PAYLOAD).max(1);
let (free_pc_est, keys_pc) = if self.is_v9() {
let runs = self.free.len() + self.pending_free.len() + 12;
let free_pc = (runs * 28 + 64).div_ceil(PAYLOAD).max(1);
let keys_pc = if self.dirty_keys {
rmp_serde::to_vec(&self.key_slots)
.map_err(|e| MnemoError::Serialize(e.to_string()))?
.len()
.div_ceil(PAYLOAD)
.max(1)
} else {
0
};
(free_pc, keys_pc)
} else {
(0, 0)
};
let bm25_seg = self.bm25_next_segment()?;
let bm25_pc = bm25_seg
.as_ref()
.map_or(0, |(b, _)| b.len().div_ceil(PAYLOAD).max(1));
let v9 = self.is_v9();
let in_wal = |pc: usize| {
if v9 && pc as u64 > V9_WAL_RUN_MAX_PAGES {
0
} else {
pc
}
};
self.ensure_wal_capacity(
in_wal(cat_pc)
+ in_wal(idx_pc)
+ in_wal(cache_pc)
+ in_wal(man_pc_est)
+ in_wal(free_pc_est)
+ in_wal(keys_pc)
+ in_wal(bm25_pc)
+ usize::from(bm25_pc > 0)
+ 1,
)?;
Ok(FlushPrelude {
bm25_seg,
cat_bytes,
cat_delta,
idx_delta,
idx_bytes,
cache_bytes,
})
}
pub fn close(&mut self) -> Result<()> {
if self.read_only {
return Ok(());
}
self.index_persist.forced = true;
self.flush()
}
pub fn flush_index(&mut self) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
self.index_persist.forced = true;
self.flush()
}
#[doc(hidden)]
pub fn __crash_partial_flush_for_testing(&mut self) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if !self.dirty_catalog && !self.dirty_index {
return Ok(());
}
self.ensure_gen_lease()?;
self.pack_records()?;
self.pack_vectors()?;
let _prelude = self.prepare_for_flush(self.dirty_index)?;
self.pager.flush()
}
pub fn set_cache_capacity(&mut self, pages: usize) {
self.pager.set_cache_capacity(pages);
}
pub fn set_vector_cache_budget(&mut self, bytes: usize) {
self.vector_cache_budget = bytes;
let need = self.catalog.len() * veccol::entry_len(self.dimensions);
if self.col.resident && need > bytes {
self.col.data = Vec::new();
self.col.norms = Vec::new();
self.col.have = Vec::new();
self.col.resident = false;
}
}
pub fn page_cache_stats(&self) -> (usize, usize) {
self.pager.cache_stats()
}
pub fn cache_put(
&mut self,
namespace: &str,
key: &str,
value: &[u8],
opts: CachePutOpts,
) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let key_hash = result_cache::hash_key(key);
let key_preview = result_cache::key_preview(key);
let now = memory::now_secs();
let size_bytes = value.len() as u64;
let entry = CacheEntry {
namespace: namespace.to_string(),
key_hash,
key_preview: key_preview.clone(),
value: value.to_vec(),
content_type: opts.content_type.clone(),
created_at: now,
accessed_at: now,
access_count: 0,
ttl_secs: opts.ttl_secs,
cost_hint_ms: opts.cost_hint_ms,
size_bytes,
vector: None,
model: None,
};
let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
let (start_page, page_count, gen) = self.write_record(&bytes)?;
if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
self.cache_dir[prev_pos].deleted = true;
self.release_cache_pages(prev_pos);
self.cache_idx.remove(namespace, &key_hash);
}
let dir_entry = CacheDirectoryEntry {
namespace: namespace.to_string(),
key_hash,
start_page,
page_count,
len: bytes.len() as u32,
deleted: false,
accessed_at: now,
access_count: 0,
size_bytes,
ttl_secs: opts.ttl_secs,
created_at: now,
vector: None,
model: None,
gen,
};
let pos = self.cache_dir.len();
self.cache_dir.push(dir_entry);
self.cache_idx.insert(namespace, key_hash, pos);
self.evict_to_budget(namespace)?;
self.dirty_cache = true;
self.cache_batch.record_dirty();
if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
Ok(())
}
pub fn cache_get(&mut self, namespace: &str, key: &str) -> Result<Option<CachedValue>> {
if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
let key_hash = result_cache::hash_key(key);
let pos = match self.cache_idx.get(namespace, &key_hash) {
Some(p) => p,
None => {
self.cache_counters.misses += 1;
return Ok(None);
}
};
let now = memory::now_secs();
let (start_page, page_count, len, ttl_secs, expired, created_at, gen) = {
let e = &self.cache_dir[pos];
(
e.start_page,
e.page_count,
e.len,
e.ttl_secs,
e.is_expired(now),
e.created_at,
e.gen,
)
};
if expired {
self.cache_counters.misses += 1;
return Ok(None);
}
let buf = self.read_record_run(start_page, page_count, len, gen)?;
let entry: CacheEntry =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if !self.read_only {
self.cache_dir[pos].accessed_at = now;
self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
self.dirty_cache = true;
}
self.cache_counters.hits += 1;
let access_count = self.cache_dir[pos].access_count;
let accessed_at = if self.read_only {
self.cache_dir[pos].accessed_at
} else {
now
};
Ok(Some(CachedValue {
value: entry.value,
content_type: entry.content_type,
created_at,
accessed_at,
access_count,
ttl_secs,
}))
}
pub fn cache_delete(&mut self, namespace: &str, key: &str) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let key_hash = result_cache::hash_key(key);
if let Some(pos) = self.cache_idx.get(namespace, &key_hash) {
self.cache_dir[pos].deleted = true;
self.release_cache_pages(pos);
self.cache_idx.remove(namespace, &key_hash);
self.dirty_cache = true;
self.cache_batch.record_dirty();
if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
}
Ok(())
}
pub fn cache_purge(&mut self, namespace: Option<&str>, expired_only: bool) -> Result<usize> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let now = memory::now_secs();
let mut purged = 0usize;
for pos in 0..self.cache_dir.len() {
if self.cache_dir[pos].deleted {
continue;
}
if let Some(ns) = namespace {
if self.cache_dir[pos].namespace != ns {
continue;
}
}
if expired_only && !self.cache_dir[pos].is_expired(now) {
continue;
}
let key_hash = self.cache_dir[pos].key_hash;
let ns = self.cache_dir[pos].namespace.clone();
self.cache_dir[pos].deleted = true;
self.release_cache_pages(pos);
self.cache_idx.remove(&ns, &key_hash);
purged += 1;
}
if purged > 0 {
self.dirty_cache = true;
self.cache_batch.record_dirty();
if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
}
Ok(purged)
}
pub fn cache_stats(&self, namespace: Option<&str>) -> CacheStats {
let mut entries = 0usize;
let mut bytes = 0u64;
for e in &self.cache_dir {
if e.deleted {
continue;
}
if let Some(ns) = namespace {
if e.namespace != ns {
continue;
}
}
entries += 1;
bytes += e.size_bytes;
}
let hits = self.cache_counters.hits;
let misses = self.cache_counters.misses;
let total = hits + misses;
let hit_rate = if total > 0 {
hits as f64 / total as f64
} else {
0.0
};
CacheStats {
entries,
bytes,
hits,
misses,
hit_rate,
evictions: self.cache_counters.evictions,
}
}
pub fn set_cache_budget(&mut self, namespace: &str, budget: CacheBudget) {
self.cache_budgets.insert(namespace.to_string(), budget);
}
pub fn set_cache_flush_policy(&mut self, policy: CacheFlushPolicy) {
self.cache_flush_policy = policy;
}
fn release_cache_pages(&mut self, pos: usize) {
let (p, c) = (
self.cache_dir[pos].start_page,
self.cache_dir[pos].page_count as u64,
);
self.release_pages(p, c);
}
fn evict_to_budget(&mut self, namespace: &str) -> Result<()> {
let budget = self
.cache_budgets
.get(namespace)
.copied()
.unwrap_or_default();
loop {
let mut live_count = 0usize;
let mut live_bytes = 0u64;
for e in &self.cache_dir {
if !e.deleted && e.namespace == namespace {
live_count += 1;
live_bytes += e.size_bytes;
}
}
if live_count <= budget.max_entries && live_bytes <= budget.max_bytes {
return Ok(());
}
let victim_pos = self
.cache_dir
.iter()
.enumerate()
.filter(|(_, e)| !e.deleted && e.namespace == namespace)
.min_by_key(|(_, e)| e.accessed_at)
.map(|(i, _)| i);
let Some(pos) = victim_pos else {
return Ok(());
};
let key_hash = self.cache_dir[pos].key_hash;
let ns = self.cache_dir[pos].namespace.clone();
self.cache_dir[pos].deleted = true;
self.release_cache_pages(pos);
self.cache_idx.remove(&ns, &key_hash);
self.cache_counters.evictions += 1;
}
}
pub fn cache_put_semantic(
&mut self,
namespace: &str,
key: &str,
vector: Vec<f32>,
value: &[u8],
opts: SemanticCachePutOpts,
) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
if vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: vector.len(),
});
}
let key_hash = result_cache::hash_key(key);
let key_preview = result_cache::key_preview(key);
let now = memory::now_secs();
let size_bytes = value.len() as u64;
let entry = CacheEntry {
namespace: namespace.to_string(),
key_hash,
key_preview: key_preview.clone(),
value: value.to_vec(),
content_type: opts.content_type.clone(),
created_at: now,
accessed_at: now,
access_count: 0,
ttl_secs: opts.ttl_secs,
cost_hint_ms: opts.cost_hint_ms,
size_bytes,
vector: Some(vector.clone()),
model: Some(opts.model.clone()),
};
let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
let (start_page, page_count, gen) = self.write_record(&bytes)?;
if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
self.cache_dir[prev_pos].deleted = true;
self.release_cache_pages(prev_pos);
self.cache_idx.remove(namespace, &key_hash);
}
let dir_entry = CacheDirectoryEntry {
namespace: namespace.to_string(),
key_hash,
start_page,
page_count,
len: bytes.len() as u32,
deleted: false,
accessed_at: now,
access_count: 0,
size_bytes,
ttl_secs: opts.ttl_secs,
created_at: now,
vector: Some(vector),
model: Some(opts.model),
gen,
};
let pos = self.cache_dir.len();
self.cache_dir.push(dir_entry);
self.cache_idx.insert(namespace, key_hash, pos);
self.evict_to_budget(namespace)?;
self.dirty_cache = true;
self.cache_batch.record_dirty();
if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
Ok(())
}
pub fn cache_get_semantic(
&mut self,
namespace: &str,
query_vector: &[f32],
threshold: f32,
model: &str,
) -> Result<Option<(CachedValue, f32)>> {
if query_vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: query_vector.len(),
});
}
if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
self.flush()?;
}
let now = memory::now_secs();
let mut best: Option<(usize, f32)> = None;
for (i, e) in self.cache_dir.iter().enumerate() {
if e.deleted || e.namespace != namespace || e.is_expired(now) {
continue;
}
let Some(ref v) = e.vector else { continue };
let Some(ref m) = e.model else { continue };
if m != model {
continue;
}
let sim = memory::cosine(query_vector, v);
if best.map(|(_, s)| sim > s).unwrap_or(true) {
best = Some((i, sim));
}
}
let Some((pos, sim)) = best else {
self.cache_counters.misses += 1;
return Ok(None);
};
if sim < threshold {
self.cache_counters.misses += 1;
return Ok(None);
}
let (start_page, page_count, len, ttl_secs, created_at, gen) = {
let e = &self.cache_dir[pos];
(
e.start_page,
e.page_count,
e.len,
e.ttl_secs,
e.created_at,
e.gen,
)
};
let buf = self.read_record_run(start_page, page_count, len, gen)?;
let entry: CacheEntry =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if !self.read_only {
self.cache_dir[pos].accessed_at = now;
self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
self.dirty_cache = true;
}
self.cache_counters.hits += 1;
let access_count = self.cache_dir[pos].access_count;
let accessed_at = if self.read_only {
self.cache_dir[pos].accessed_at
} else {
now
};
Ok(Some((
CachedValue {
value: entry.value,
content_type: entry.content_type,
created_at,
accessed_at,
access_count,
ttl_secs,
},
sim,
)))
}
pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
crate::session::Session::new(self, agent_id.into())
}
pub fn snapshots(&self) -> Vec<SnapshotInfo> {
let mut v: Vec<SnapshotInfo> = self
.manifest
.iter()
.map(|s| SnapshotInfo {
txn_id: s.txn_id,
created_at: s.created_at,
memory_count: s.memory_count,
})
.collect();
v.sort_by_key(|s| s.txn_id);
v
}
fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
let catalog = load_catalog(
&mut self.pager,
snap.catalog_start,
snap.catalog_pages,
snap.catalog_len,
snap.catalog_gen(),
)?;
let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
self.catalog_rewrite = true;
let ann = load_index(
&mut self.pager,
snap.index_start,
snap.index_pages,
snap.index_len,
snap.index_gen(),
self.dimensions,
)?;
let mut ann = ann;
if self.is_v9() {
self.reclaim_for_restore(&catalog);
if let Some(a) = ann.as_mut() {
apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
a.enable_log();
}
self.index_rebase = true;
let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
let runs = |ptr: RunPtr, m: &Bm25Manifest| {
let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
if !ptr.is_empty() {
v.push((ptr.start, ptr.pages));
}
v
};
let then = runs(bptr, &manifest);
let now = runs(self.header.bm25, &self.bm25.manifest);
self.pending_free.retain(|r| !then.contains(r));
for &(p, c) in &then {
self.free.unfree(p, c);
}
for r in now {
if !then.contains(&r) {
self.release_pages(r.0, r.1);
}
}
self.header.bm25 = bptr;
self.free_dirty = true;
self.bm25 = Bm25State {
manifest,
..Bm25State::default()
};
}
self.index = build_id_index(&catalog);
self.catalog = catalog;
self.col = VecColumn::from_catalog(&self.catalog);
self.recs = RecPages::from_catalog(&self.catalog);
self.ann = ann;
self.ann_unloaded = false;
self.key_index = None;
self.superseded = None;
self.layers = None;
self.agent_set = None;
let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
self.dirty_catalog = true;
self.dirty_index = true;
self.index_persist.forced = true;
self.flush()
}
fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
if self.snapshot_cap_explicit {
if self.max_snapshots > 0 {
while manifest.len() > self.max_snapshots {
manifest.remove(0);
}
}
return;
}
let now = memory::now_secs();
if self.max_snapshots > 0 {
while manifest.len() > self.max_snapshots
&& now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
{
manifest.remove(0);
}
}
let live_pages = self.header.next_page.saturating_sub(self.free.pages());
let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
while manifest.len() > 1 {
let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
if pinned <= cap {
break;
}
manifest.remove(0);
}
}
fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
cat.iter()
.filter(|e| !e.deleted && e.start_page != 0)
.map(|e| (e.start_page, e.page_count as u64))
.collect()
};
let live_then = runs(restored);
let live_now = runs(&self.catalog);
let overlaps = |p: u64, c: u64| {
live_then
.range(..p + c)
.next_back()
.is_some_and(|(&q, &n)| q + n > p)
};
let dying: Vec<(u64, u64)> = live_now
.iter()
.filter(|(p, _)| !live_then.contains_key(p))
.map(|(&p, &c)| (p, c))
.collect();
let vec_then: std::collections::BTreeSet<u64> = restored
.iter()
.filter(|e| !e.deleted)
.filter_map(|e| e.v9.map(|r| r.vec_page))
.filter(|&p| p != 0)
.collect();
let vec_dying: Vec<u64> = self
.col
.page_rows
.keys()
.copied()
.filter(|p| !vec_then.contains(p))
.collect();
self.pending_free
.retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
for (&p, &c) in &live_then {
self.free.unfree(p, c);
}
for &p in &vec_then {
self.free.unfree(p, 1);
}
for (p, c) in dying {
self.release_pages(p, c);
}
for p in vec_dying {
self.release_pages(p, 1);
}
self.free_dirty = true;
}
pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let snap = self
.manifest
.iter()
.find(|s| s.txn_id == txn_id)
.cloned()
.ok_or_else(|| {
let oldest = self.manifest.iter().map(|s| s.txn_id).min();
match oldest {
Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
"snapshot for transaction {txn_id}: older than the retention window \
(the newest {} snapshots and any from the last 24 hours, with at \
most a quarter of the file pinned); the oldest kept is {o}",
self.max_snapshots
)),
_ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
}
})?;
self.apply_snapshot(&snap)?;
Ok(SnapshotInfo {
txn_id: snap.txn_id,
created_at: snap.created_at,
memory_count: snap.memory_count,
})
}
pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let snap = self
.manifest
.iter()
.filter(|s| s.created_at <= unix_secs)
.max_by_key(|s| s.txn_id)
.cloned()
.ok_or_else(|| {
MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
})?;
self.apply_snapshot(&snap)?;
Ok(SnapshotInfo {
txn_id: snap.txn_id,
created_at: snap.created_at,
memory_count: snap.memory_count,
})
}
pub fn kdf_params(&self) -> KdfParams {
self.kdf
}
pub fn export_encrypted(&mut self, dest: impl AsRef<Path>) -> Result<()> {
let dest = dest.as_ref();
if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
return Err(MnemoError::Invalid(format!(
"export_encrypted refuses to overwrite the open database at {}",
dest.display()
)));
}
if !self.read_only {
self.flush()?;
}
let parent = dest.parent().ok_or_else(|| {
MnemoError::Invalid(format!(
"export_encrypted: dest {} has no parent directory",
dest.display()
))
})?;
let tmp = temp_sibling(dest);
{
let src = self.pager.file_mut();
src.seek(SeekFrom::Start(0))?;
let mut out = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&tmp)?;
let mut buf = vec![0u8; 1 << 20]; loop {
let n = src.read(&mut buf)?;
if n == 0 {
break;
}
out.write_all(&buf[..n])?;
}
out.sync_all()?;
crate::bump_fsync();
}
if let Err(e) = std::fs::rename(&tmp, dest) {
let _ = std::fs::remove_file(&tmp);
return Err(MnemoError::Io(e));
}
#[cfg(unix)]
{
if let Ok(dir) = File::open(parent) {
let _ = dir.sync_all();
crate::bump_fsync();
}
}
#[cfg(not(unix))]
{
let _ = parent;
}
Ok(())
}
pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
self.rekey_secret(new_passphrase.as_bytes(), kdf)
}
pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
self.rekey_secret(new_key, KdfParams::raw_key())
}
fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
self.flush()?;
let new_salt = crypto::random_salt();
let new_kek = crypto::derive_kek(secret, &new_salt, kdf)?;
let new_nonce = crypto::random_nonce();
let wrapped = crypto::wrap_dek(&new_kek, &new_nonce, self.pager.dek())?;
if wrapped.len() != WRAPPED_DEK_LEN {
return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
}
let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
wrapped_dek.copy_from_slice(&wrapped);
self.header.salt = new_salt;
self.header.dek_nonce = new_nonce;
self.header.wrapped_dek = wrapped_dek;
self.header.m_cost = kdf.m_cost;
self.header.t_cost = kdf.t_cost;
self.header.p_cost = kdf.p_cost;
self.kdf = kdf;
let old_wal_start = self.header.wal_start;
let new_pages = self.header.wal_pages;
let new_start = self.header.next_page;
self.header.next_page += new_pages;
self.header.wal_start = new_start;
self.header.wal_pages = new_pages;
self.header.wal_seq += 1;
self.relocate_header_through_wal(old_wal_start)?;
self.release_pages(old_wal_start, new_pages);
Ok(())
}
pub fn verify(&mut self) -> Result<usize> {
let entries: Vec<CatalogEntry> = self
.catalog
.iter()
.filter(|e| !e.deleted)
.cloned()
.collect();
let mut n = entries.len();
for e in &entries {
let bytes = self.read_record(e)?;
let _m: Memory = self.decode_memory(&bytes)?;
if e.v9.is_some() {
let (h, _) = self.column_entry(e)?;
if h.len() != self.dimensions {
return Err(MnemoError::Invalid(
"vector entry has the wrong width".into(),
));
}
}
}
let cache_positions: Vec<(u64, u32, u32, u64)> = self
.cache_dir
.iter()
.filter(|e| !e.deleted)
.map(|e| (e.start_page, e.page_count, e.len, e.gen))
.collect();
for (start_page, page_count, len, gen) in cache_positions {
let buf = self.read_record_run(start_page, page_count, len, gen)?;
let _c: CacheEntry = rmp_serde::from_slice(&buf)
.map_err(|err| MnemoError::Serialize(err.to_string()))?;
n += 1;
}
if self.is_v9() {
self.check_free_list_disjoint()?;
}
Ok(n)
}
fn check_free_list_disjoint(&self) -> Result<()> {
let h = &self.header;
let mut live: Vec<(u64, u64, &str)> = vec![
(h.wal_start, h.wal_pages, "WAL region"),
(h.catalog_start, h.catalog_pages, "catalog run"),
(h.index_start, h.index_pages, "index run"),
(h.manifest_start, h.manifest_pages, "manifest run"),
(h.cache_start, h.cache_pages, "cache directory run"),
(h.free.start, h.free.pages, "free-list run"),
(h.keys.start, h.keys.pages, "key-slot run"),
];
live.extend(
self.catalog
.iter()
.filter(|e| !e.deleted)
.map(|e| (e.start_page, e.page_count as u64, "record")),
);
live.extend(
self.cache_dir
.iter()
.filter(|e| !e.deleted)
.map(|e| (e.start_page, e.page_count as u64, "cache entry")),
);
live.extend(self.col.page_rows.keys().map(|&p| (p, 1, "vector page")));
live.extend(
self.delta_chain
.iter()
.map(|r| (r.start, r.pages, "catalog delta run")),
);
live.extend(
self.index_chain
.iter()
.map(|r| (r.start, r.pages, "index delta run")),
);
live.push((
self.header.bm25.start,
self.header.bm25.pages,
"BM25 manifest",
));
live.extend(
self.bm25
.manifest
.segments
.iter()
.map(|t| (t.0, t.1, "BM25 segment")),
);
for (p, c, what) in live {
if c == 0 {
continue;
}
if self.free.overlaps(p, c) {
return Err(MnemoError::Invalid(format!(
"free list overlaps a live {what} at page {p} (+{c})"
)));
}
if self
.pending_free
.iter()
.any(|&(q, n)| q < p + c && p < q + n)
{
return Err(MnemoError::Invalid(format!(
"a pending free overlaps a live {what} at page {p} (+{c})"
)));
}
}
Ok(())
}
pub fn stats(&mut self) -> Result<Stats> {
let deleted = self.catalog.iter().filter(|e| e.deleted).count();
if self.agent_set.is_none() {
let built: std::collections::BTreeSet<String> =
self.memories()?.into_iter().map(|m| m.agent_id).collect();
self.agent_set = Some(built);
}
let agents: Vec<String> = self
.agent_set
.as_ref()
.expect("agent_set just filled above")
.iter()
.cloned()
.collect();
let file_bytes = self.header.next_page * PAGE_SIZE as u64;
Ok(Stats {
memories: self.len(),
deleted,
dimensions: self.dimensions,
file_bytes,
agents,
encrypted: self.header.flags & FLAG_ENCRYPTED != 0,
created_at: self.header.created_at,
index: {
self.ensure_ann()?;
self.ann.as_ref().map(|a| a.info())
},
wal_pages: self.header.wal_pages,
fsyncs_last_flush: self.fsyncs_last_flush,
format_version: self.header.version,
})
}
pub fn compact_file(path: &str, passphrase: &str) -> Result<CompactReport> {
Self::compact_file_secret(path, passphrase.as_bytes(), 0)
}
pub fn compact_file_with_key(path: &str, key: &[u8; crypto::KEY_LEN]) -> Result<CompactReport> {
Self::compact_file_secret(path, key, 0)
}
pub fn compact_file_to(
path: &str,
passphrase: &str,
format_version: u16,
) -> Result<CompactReport> {
if format_version != VERSION && format_version != VERSION_V9 {
return Err(MnemoError::Invalid(format!(
"compact target format {format_version} isn't supported (use 8 or 9)"
)));
}
Self::compact_file_secret(path, passphrase.as_bytes(), format_version)
}
fn compact_file_secret(
path: &str,
secret: &[u8],
format_version: u16,
) -> Result<CompactReport> {
let mut old = Mnemo::open_secret(path, secret)?;
let dims = old.dimensions;
let kdf = old.kdf;
let tmp = format!("{path}.compact-tmp");
let mut new = Mnemo::create_secret(
&tmp,
secret,
MnemoConfig {
dimensions: dims,
kdf,
format_version,
store_float_vectors: old.float_vectors(),
..Default::default()
},
)?;
new.turn_compress_after = 0;
let want_index = old.has_index();
let now = memory::now_secs();
let all = old.memories()?;
let before = all.len();
let mut after = 0;
for m in all {
if m.is_expired(now) {
continue;
}
new.remember(m)?;
after += 1;
}
if want_index && after > 0 {
new.build_index_with(IndexConfig::default())?;
}
let mut cache_before = 0usize;
let mut cache_after = 0usize;
let positions: Vec<usize> = (0..old.cache_dir.len()).collect();
for pos in positions {
let keep = {
let e = &old.cache_dir[pos];
!e.deleted && !e.is_expired(now)
};
if keep {
cache_before += 1;
new.copy_cache_entry_from(&mut old, pos)?;
cache_after += 1;
}
}
new.flush()?;
drop(new);
drop(old);
std::fs::rename(&tmp, path)?;
Ok(CompactReport {
before,
after,
cache_before,
cache_after,
})
}
pub(crate) fn copy_cache_entry_from(&mut self, old: &mut Mnemo, pos: usize) -> Result<()> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let (start_page, page_count, len, dir_accessed_at, dir_access_count, old_gen) = {
let e = &old.cache_dir[pos];
(
e.start_page,
e.page_count,
e.len,
e.accessed_at,
e.access_count,
e.gen,
)
};
let buf = old.read_record_run(start_page, page_count, len, old_gen)?;
let mut entry: CacheEntry =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
entry.accessed_at = dir_accessed_at;
entry.access_count = dir_access_count;
let bytes =
rmp_serde::to_vec(&entry).map_err(|err| MnemoError::Serialize(err.to_string()))?;
let (new_start_page, new_page_count, gen) = self.write_record(&bytes)?;
let dir_entry = CacheDirectoryEntry {
namespace: entry.namespace.clone(),
key_hash: entry.key_hash,
start_page: new_start_page,
page_count: new_page_count,
len: bytes.len() as u32,
deleted: false,
accessed_at: dir_accessed_at,
access_count: dir_access_count,
size_bytes: entry.size_bytes,
ttl_secs: entry.ttl_secs,
created_at: entry.created_at,
vector: entry.vector.clone(),
model: entry.model.clone(),
gen,
};
let new_pos = self.cache_dir.len();
self.cache_idx
.insert(&entry.namespace, entry.key_hash, new_pos);
self.cache_dir.push(dir_entry);
self.dirty_cache = true;
Ok(())
}
}