use super::*;
pub(super) const REC_SPAN: u16 = u16::MAX;
pub(super) const PAGE_KIND_RECORD: u8 = 1;
pub(super) const REC_PAGE_CAP: usize = PAYLOAD - veccol::PAGE_HEADER;
pub(super) const REC_SLOT_BYTES: usize = 4;
pub(super) const V9_TURN_CHUNK_BYTES: usize = 7_900;
pub(super) const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD];
pub(super) const TURN_ZSTD_LEVEL: i32 = 3;
pub(super) const TURN_ZSTD_MIN_BYTES: usize = 256;
pub(super) const RECORD_PLAIN_MAX: u64 = 1 << 20;
pub(super) const COLD_TURNS_PER_COMMIT: usize = 256;
#[cfg(feature = "dev-hooks")]
pub(super) fn turn_zstd_enabled() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("MNEMO_DEV_TURN_ZSTD").as_deref() != Ok("0"))
}
#[cfg(not(feature = "dev-hooks"))]
pub(super) fn turn_zstd_enabled() -> bool {
true
}
pub(super) fn compress_turn_record(bytes: Vec<u8>) -> Vec<u8> {
if bytes.len() < TURN_ZSTD_MIN_BYTES || !turn_zstd_enabled() {
return bytes;
}
let c = ZSTD_COMPRESSOR.with(|cell| {
let mut slot = cell.borrow_mut();
if slot.is_none() {
*slot = zstd::bulk::Compressor::new(TURN_ZSTD_LEVEL).ok();
}
let out = slot.as_mut().and_then(|z| z.compress(&bytes).ok());
if out.is_none() {
*slot = None;
}
out
});
match c {
Some(c) if c.len() * 8 <= bytes.len() * 7 => c,
_ => bytes,
}
}
thread_local! {
static ZSTD_COMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Compressor<'static>>> =
const { std::cell::RefCell::new(None) };
static ZSTD_DECOMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Decompressor<'static>>> =
const { std::cell::RefCell::new(None) };
}
pub(crate) fn plain_record(bytes: &[u8]) -> Result<std::borrow::Cow<'_, [u8]>> {
if !bytes.starts_with(&ZSTD_MAGIC) {
return Ok(std::borrow::Cow::Borrowed(bytes));
}
let bad = |why: &str| MnemoError::Invalid(format!("compressed record: {why}"));
let size = match zstd::zstd_safe::get_frame_content_size(bytes) {
Ok(Some(n)) if n <= RECORD_PLAIN_MAX => n as usize,
Ok(Some(_)) => return Err(bad("too large")),
_ => return Err(bad("no content size")),
};
let out = ZSTD_DECOMPRESSOR.with(|cell| {
let mut slot = cell.borrow_mut();
if slot.is_none() {
*slot = Some(zstd::bulk::Decompressor::new().map_err(|e| bad(&e.to_string()))?);
}
let out = match slot.as_mut() {
Some(z) => z.decompress(bytes, size).map_err(|e| bad(&e.to_string())),
None => Err(bad("no context")),
};
if out.is_err() {
*slot = None;
}
out
})?;
if out.len() != size {
return Err(bad("length mismatch"));
}
Ok(std::borrow::Cow::Owned(out))
}
pub(super) fn vacuum_pick(sparse: Vec<(u64, usize)>) -> Vec<u64> {
let mut rows = 0usize;
let mut out = Vec::new();
for (p, n) in sparse {
if out.len() == VACUUM_PAGES_PER_COMMIT {
break;
}
if rows + n > VACUUM_MAX_ROWS {
continue;
}
rows += n;
out.push(p);
}
out
}
pub(super) fn encode_record_page(gen: u64, recs: &[&[u8]]) -> Vec<u8> {
let dir = recs.len() * REC_SLOT_BYTES;
let data: usize = recs.iter().map(|r| r.len()).sum();
let mut out = Vec::with_capacity(veccol::PAGE_HEADER + dir + data);
out.extend_from_slice(&[PAGE_KIND_RECORD, 0, 0, 0]);
out.extend_from_slice(&(recs.len() as u16).to_le_bytes());
out.extend_from_slice(&((dir + data) as u16).to_le_bytes());
out.extend_from_slice(&gen.to_le_bytes());
let mut off = veccol::PAGE_HEADER + dir;
for r in recs {
out.extend_from_slice(&(off as u16).to_le_bytes());
out.extend_from_slice(&(r.len() as u16).to_le_bytes());
off += r.len();
}
for r in recs {
out.extend_from_slice(r);
}
out
}
pub(super) fn record_slot(page: &[u8], slot: u16) -> Result<Vec<u8>> {
let bad = || MnemoError::Invalid("malformed record page".into());
if page.len() < veccol::PAGE_HEADER || page[0] != PAGE_KIND_RECORD {
return Err(bad());
}
let n = u16::from_le_bytes([page[4], page[5]]);
if slot >= n {
return Err(bad());
}
let d = veccol::PAGE_HEADER + slot as usize * REC_SLOT_BYTES;
let dir = page.get(d..d + REC_SLOT_BYTES).ok_or_else(bad)?;
let off = u16::from_le_bytes([dir[0], dir[1]]) as usize;
let len = u16::from_le_bytes([dir[2], dir[3]]) as usize;
page.get(off..off + len).map(|b| b.to_vec()).ok_or_else(bad)
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub(super) struct Bm25Manifest {
pub(super) tokenizer: u8,
pub(super) next_doc: u32,
pub(super) segments: Vec<(u64, u64, u64, u64)>,
#[serde(default)]
pub(super) seg_docs: Vec<u32>,
}
impl Default for Bm25Manifest {
fn default() -> Self {
Bm25Manifest {
tokenizer: text::TOKENIZER_V1,
next_doc: 1,
segments: Vec::new(),
seg_docs: Vec::new(),
}
}
}
#[derive(Debug, Default)]
pub(super) struct DocStats {
pub(super) docs: std::collections::HashMap<u32, (usize, u16)>,
pub(super) avgdl: f32,
pub(super) total: u64,
}
impl DocStats {
pub(super) fn remove(&mut self, doc: u32) {
if let Some((_, len)) = self.docs.remove(&doc) {
self.total = self.total.saturating_sub(len as u64);
self.refresh();
}
}
pub(super) fn insert(&mut self, doc: u32, row: usize, len: u16) {
if let Some((_, old)) = self.docs.insert(doc, (row, len)) {
self.total = self.total.saturating_sub(old as u64);
}
self.total += len as u64;
self.refresh();
}
pub(super) fn refresh(&mut self) {
self.avgdl = if self.docs.is_empty() {
1.0
} else {
self.total as f32 / self.docs.len() as f32
};
}
}
#[derive(Debug, Default)]
pub(super) struct Bm25State {
pub(super) manifest: Bm25Manifest,
pub(super) dicts: Option<Vec<Vec<text::DictEntry>>>,
pub(super) pending: Vec<(u32, Vec<(String, u8)>)>,
pub(super) stats: Option<DocStats>,
pub(super) merged_docs: u32,
}
pub(super) type Bm25Segments = (Vec<(u64, u64, u64, u64)>, Vec<u32>);
pub(super) const BM25_MAX_SEGMENTS: usize = 8;
pub(super) fn scored_row(e: &CatalogEntry) -> bool {
!e.deleted
&& e.v9.is_some_and(|r| {
r.flags & ROW_SUPERSEDED == 0
&& !matches!(
r.kind,
ROW_KIND_TURN | ROW_KIND_ESSENCE | ROW_KIND_THREAD_LINK
)
})
}
pub(super) fn bm25_kind(kind: u8) -> bool {
kind == ROW_KIND_FACT || kind == ROW_KIND_ONBOARDING
}
pub(super) fn read_run_range(
pager: &mut Pager,
ptr: RunPtr,
off: u64,
len: u64,
) -> Result<Vec<u8>> {
let mut out = Vec::with_capacity(len as usize);
if len == 0 {
return Ok(out);
}
if off + len > ptr.len {
return Err(MnemoError::Invalid("BM25 range past its run".into()));
}
let p = PAYLOAD as u64;
let (first, last) = (off / p, (off + len - 1) / p);
for i in first..=last {
let page = pager.read_page(ptr.start + i, ptr.gen)?;
let lo = if i == first { (off % p) as usize } else { 0 };
let hi = if i == last {
((off + len - 1) % p) as usize + 1
} else {
PAYLOAD
};
out.extend_from_slice(&page[lo..hi]);
}
Ok(out)
}
pub(super) fn load_segment_dict(pager: &mut Pager, ptr: RunPtr) -> Result<Vec<text::DictEntry>> {
let head = read_run_range(pager, ptr, 0, 4)?;
let n = u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
let bytes = read_run_range(pager, ptr, 4, n)?;
rmp_serde::from_slice(&bytes).map_err(|e| MnemoError::Serialize(e.to_string()))
}
pub(super) fn segment_postings(
pager: &mut Pager,
ptr: RunPtr,
dict: &[text::DictEntry],
term: &str,
) -> Result<Vec<(u32, u8)>> {
let Ok(i) = dict.binary_search_by(|e| e.term.as_str().cmp(term)) else {
return Ok(Vec::new());
};
let head = read_run_range(pager, ptr, 0, 4)?;
let base = 4 + u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
let e = &dict[i];
let bytes = read_run_range(pager, ptr, base + e.off as u64, e.len as u64)?;
text::decode_postings(&bytes)
}
pub(super) fn load_bm25_manifest(pager: &mut Pager, ptr: RunPtr) -> Result<Bm25Manifest> {
if ptr.is_empty() {
return Ok(Bm25Manifest::default());
}
let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
let m: Bm25Manifest =
rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
if m.tokenizer != text::TOKENIZER_V1 {
return Err(MnemoError::Invalid(format!(
"BM25 tokenizer {} is newer than this build understands",
m.tokenizer
)));
}
Ok(m)
}
#[derive(Debug, Default)]
pub(super) struct RecPages {
pub(super) hot: std::collections::HashMap<usize, Vec<u8>>,
pub(super) pending: Vec<usize>,
pub(super) page_rows: std::collections::HashMap<u64, Vec<usize>>,
}
impl RecPages {
pub(super) fn from_catalog(catalog: &[CatalogEntry]) -> RecPages {
let mut recs = RecPages::default();
for (i, e) in catalog.iter().enumerate() {
if e.deleted || e.start_page == 0 {
continue;
}
if let Some(r) = e.v9 {
if r.rec_slot != REC_SPAN {
recs.page_rows.entry(e.start_page).or_default().push(i);
}
}
}
recs
}
}
pub(super) const ROW_SUPERSEDED: u8 = 1;
pub(super) const ROW_HAS_TTL: u8 = 2;
pub(super) const ROW_SHARED: u8 = 4;
pub(super) const ROW_SETTLED: u8 = 8;
pub(super) const ROW_KIND_FACT: u8 = 0;
pub(super) const ROW_KIND_TURN: u8 = 1;
pub(super) const ROW_KIND_ESSENCE: u8 = 2;
pub(super) const ROW_KIND_THREAD_LINK: u8 = 3;
pub(super) const ROW_KIND_ONBOARDING: u8 = 4;
pub(super) fn fnv1a32(parts: &[&[u8]]) -> u32 {
let mut h: u32 = 0x811c_9dc5;
for (i, p) in parts.iter().enumerate() {
if i > 0 {
h ^= 0xff;
h = h.wrapping_mul(0x0100_0193);
}
for &b in p.iter() {
h ^= b as u32;
h = h.wrapping_mul(0x0100_0193);
}
}
h.max(1)
}
pub(super) fn memory_type_code(t: MemoryType) -> u8 {
match t {
MemoryType::Episodic => 0,
MemoryType::Semantic => 1,
MemoryType::Procedural => 2,
MemoryType::Working => 3,
}
}
pub(super) fn row_tags(m: &Memory) -> RowV9 {
let kind = match meta_str(m, MNEMO_META_KIND) {
Some(MNEMO_META_KIND_TURN) => ROW_KIND_TURN,
Some(MNEMO_META_KIND_ESSENCE) => ROW_KIND_ESSENCE,
Some(MNEMO_META_KIND_THREAD_LINK) => ROW_KIND_THREAD_LINK,
_ if is_onboarding(m) => ROW_KIND_ONBOARDING,
_ => ROW_KIND_FACT,
};
let mut flags = 0u8;
if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
flags |= ROW_SUPERSEDED;
}
if m.ttl_secs.is_some() {
flags |= ROW_HAS_TTL;
}
if m.scope == Scope::Shared {
flags |= ROW_SHARED;
}
let key_hash = meta_str(m, MNEMO_META_KEY).map_or(0, |k| {
let slot = Mnemo::key_slot_for(m, k);
fnv1a32(&[
slot.scope_owner.as_deref().unwrap_or("").as_bytes(),
slot.key.as_bytes(),
])
});
RowV9 {
kind,
flags,
memory_type: memory_type_code(m.memory_type),
thread_hash: meta_str(m, MNEMO_META_THREAD).map_or(0, |t| fnv1a32(&[t.as_bytes()])),
key_hash,
agent_hash: fnv1a32(&[m.agent_id.as_bytes()]),
created_at: m.created_at,
importance: m.importance,
expires_at: m.ttl_secs.map_or(0, |t| m.created_at.saturating_add(t)),
..RowV9::default()
}
}
#[derive(Debug, Default)]
pub(super) struct VecColumn {
pub(super) data: Vec<u16>,
pub(super) norms: Vec<f32>,
pub(super) have: Vec<bool>,
pub(super) resident: bool,
pub(super) scored_rows: usize,
pub(super) hot: std::collections::HashMap<usize, (Vec<u16>, f32)>,
pub(super) pending: Vec<usize>,
pub(super) page_rows: std::collections::HashMap<u64, Vec<usize>>,
}
impl VecColumn {
pub(super) fn from_catalog(catalog: &[CatalogEntry]) -> VecColumn {
let mut col = VecColumn::default();
for (i, e) in catalog.iter().enumerate() {
if e.deleted {
continue;
}
if let Some(r) = e.v9 {
if r.vec_page != 0 {
col.page_rows.entry(r.vec_page).or_default().push(i);
}
}
}
col
}
}
impl Mnemo {
pub(super) fn is_v9(&self) -> bool {
self.header.version >= VERSION_V9
}
pub(super) fn write_gen(&self) -> u64 {
if self.is_v9() {
self.header.wal_seq + 1
} else {
0
}
}
pub(super) fn min_snapshot_txn(&self) -> u64 {
self.manifest
.iter()
.map(|s| s.txn_id)
.min()
.unwrap_or(u64::MAX)
}
pub(super) fn alloc_pages(&mut self, n: u64) -> u64 {
if self.is_v9() {
let committed = self.header.wal_seq;
let min_snap = self.min_snapshot_txn();
if let Some(p) = self.free.alloc(n, committed, min_snap) {
self.free_dirty = true;
return p;
}
}
let p = self.header.next_page;
self.header.next_page += n;
p
}
pub(super) fn release_pages(&mut self, page: u64, n: u64) {
if self.is_v9() && n > 0 && page != 0 {
self.pending_free.push((page, n));
}
}
pub(super) fn ensure_bm25(&mut self) -> Result<()> {
self.ensure_bm25_stats();
self.ensure_bm25_dicts()
}
pub(super) fn ensure_bm25_stats(&mut self) {
if self.bm25.stats.is_none() {
let mut docs = std::collections::HashMap::new();
let mut total = 0u64;
for (i, e) in self.catalog.iter().enumerate() {
if e.deleted {
continue;
}
if let Some(r) = e.v9 {
if r.doc != 0 && bm25_kind(r.kind) {
docs.insert(r.doc, (i, r.doc_len));
total += r.doc_len as u64;
}
}
}
let mut stats = DocStats {
docs,
avgdl: 1.0,
total,
};
stats.refresh();
self.bm25.stats = Some(stats);
}
}
pub(super) fn ensure_bm25_dicts(&mut self) -> Result<()> {
if self.bm25.dicts.is_none() {
let mut dicts = Vec::with_capacity(self.bm25.manifest.segments.len());
for &t in &self.bm25.manifest.segments {
dicts.push(load_segment_dict(&mut self.pager, tuple_ptr(t))?);
}
self.bm25.dicts = Some(dicts);
}
Ok(())
}
pub(super) fn bm25_scores(
&mut self,
query: &str,
k1: f32,
b: f32,
) -> Result<std::collections::HashMap<usize, f32>> {
let mut scores: std::collections::HashMap<usize, f32> = std::collections::HashMap::new();
if !self.is_v9() {
return Ok(scores);
}
self.ensure_bm25()?;
let terms: std::collections::BTreeSet<String> = text::tokenize(query).into_iter().collect();
let segs: Vec<RunPtr> = self
.bm25
.manifest
.segments
.iter()
.map(|&t| tuple_ptr(t))
.collect();
let stats = self.bm25.stats.as_ref().expect("loaded above");
let dicts = self.bm25.dicts.as_ref().expect("loaded above");
let n = stats.docs.len();
for term in &terms {
let mut posts: Vec<(u32, u8)> = Vec::new();
for (si, seg) in segs.iter().enumerate() {
posts.extend(segment_postings(&mut self.pager, *seg, &dicts[si], term)?);
}
for (doc, tfs) in &self.bm25.pending {
if let Ok(i) = tfs.binary_search_by(|(t, _)| t.as_str().cmp(term)) {
posts.push((*doc, tfs[i].1));
}
}
posts.retain(|(d, _)| stats.docs.contains_key(d));
if posts.is_empty() {
continue;
}
let idf = text::idf(n, posts.len());
for (d, tf) in posts {
let (row, len) = stats.docs[&d];
*scores.entry(row).or_default() += text::bm25(tf, idf, len, stats.avgdl, k1, b);
}
}
Ok(scores)
}
pub(super) fn bm25_next_segment(&mut self) -> Result<Option<(Vec<u8>, Option<usize>)>> {
if !self.is_v9() || self.bm25.pending.is_empty() {
return Ok(None);
}
let segs: Vec<RunPtr> = self
.bm25
.manifest
.segments
.iter()
.map(|&t| tuple_ptr(t))
.collect();
self.ensure_bm25_stats();
let written: u64 = self.bm25.manifest.seg_docs.iter().map(|&d| d as u64).sum();
let live_docs = self.bm25.stats.as_ref().map_or(0, |s| s.docs.len()) as u64;
let mostly_dead = !segs.is_empty() && written > 64 && live_docs * 2 < written;
if segs.len() < BM25_MAX_SEGMENTS && !mostly_dead {
return Ok(Some((text::build_segment(&self.bm25.pending)?, None)));
}
let small: u64 = segs.iter().skip(1).map(|p| p.len).sum();
let from = if mostly_dead || small * 10 > segs[0].len {
0
} else {
1
};
let live: std::collections::HashSet<u32> = self
.bm25
.stats
.as_ref()
.expect("loaded above")
.docs
.keys()
.copied()
.collect();
let mut by_doc: std::collections::BTreeMap<u32, Vec<(String, u8)>> =
std::collections::BTreeMap::new();
for seg in &segs[from..] {
let bytes = read_run_bytes(&mut self.pager, seg.start, seg.pages, seg.len, seg.gen)?;
let n = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
let dict: Vec<text::DictEntry> = rmp_serde::from_slice(&bytes[4..4 + n])
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
let post = &bytes[4 + n..];
for e in dict {
let range = e.off as usize..(e.off + e.len) as usize;
let list = text::decode_postings(post.get(range).ok_or_else(|| {
MnemoError::Invalid("BM25 dict entry past its postings".into())
})?)?;
for (d, tf) in list {
if live.contains(&d) {
by_doc.entry(d).or_default().push((e.term.clone(), tf));
}
}
}
}
for (d, tfs) in &self.bm25.pending {
if live.contains(d) {
by_doc.insert(*d, tfs.clone());
}
}
let docs: Vec<(u32, Vec<(String, u8)>)> = by_doc.into_iter().collect();
self.bm25.merged_docs = docs.len() as u32;
Ok(Some((text::build_segment(&docs)?, Some(from))))
}
pub(super) fn index_gen_now(&self) -> u64 {
self.header.index_gen.max(self.header.index_delta.gen)
}
pub(super) fn plan_index_v9(&mut self) -> bool {
self.index_delta_ops = None;
let Some(ann) = self.ann.as_mut() else {
return self.dirty_index;
};
let ops = ann.take_log();
let n = ann.len();
if self.index_rebase {
return true;
}
if ops.is_empty() {
return false;
}
if self.index_chain.len() >= 64 || (self.index_chain_ops + ops.len()) * 4 > n.max(256) {
return true;
}
self.index_delta_ops = Some(ops);
true
}
pub(super) fn touch(&mut self, idx: usize) {
if self.is_v9() {
self.dirty_rows.insert(idx);
}
}
pub(super) fn catalog_as_delta(&self) -> bool {
self.is_v9()
&& !self.catalog_rewrite
&& self.header.catalog_pages > 0
&& self.delta_chain.len() < 64
&& (self.delta_rows + self.dirty_rows.len()) * 4 <= self.catalog.len()
}
pub(super) fn compact_tombstones(&mut self) {
let dead = self.catalog.iter().filter(|e| e.deleted).count();
if dead == 0 || dead * 4 <= self.catalog.len() {
return;
}
self.catalog.retain(|e| !e.deleted);
self.catalog.shrink_to_fit();
self.index = build_id_index(&self.catalog);
self.col = VecColumn::from_catalog(&self.catalog);
self.recs = RecPages::from_catalog(&self.catalog);
self.dirty_rows.clear();
self.bm25.stats = None;
}
pub(super) fn float_vectors(&self) -> bool {
self.is_v9() && self.header.flags & FLAG_FLOAT_VECTORS != 0
}
pub(super) fn stage_vector(&mut self, idx: usize, v: &[f32]) {
let (h, norm) = veccol::quantize(v);
let d = self.dimensions;
if self.col.resident {
let rows = self.catalog.len();
let new_scored = self.catalog.get(idx).is_some_and(scored_row)
&& !self.col.have.get(idx).copied().unwrap_or(false);
let scored = self.col.scored_rows + usize::from(new_scored);
let fits = scored * veccol::entry_len(d) <= self.vector_cache_budget;
if fits {
self.col.scored_rows = scored;
if self.col.data.len() < rows * d {
self.col.data.resize(rows * d, 0);
self.col.norms.resize(rows, 0.0);
self.col.have.resize(rows, false);
}
self.col.data[idx * d..(idx + 1) * d].copy_from_slice(&h);
self.col.norms[idx] = norm;
self.col.have[idx] = true;
} else {
self.col.data = Vec::new();
self.col.norms = Vec::new();
self.col.have = Vec::new();
self.col.resident = false;
}
}
self.col.hot.insert(idx, (h, norm));
self.col.pending.push(idx);
self.touch(idx);
if let Some(r) = self.catalog[idx].v9.as_mut() {
r.vec_page = 0;
r.vec_slot = 0;
r.vec_gen = 0;
}
}
pub(super) fn kill_vector(&mut self, idx: usize, row: &CatalogEntry) {
self.col.hot.remove(&idx);
let Some(r) = row.v9 else { return };
if r.vec_page == 0 {
return;
}
let empty = match self.col.page_rows.get_mut(&r.vec_page) {
Some(rows) => {
rows.retain(|&i| i != idx);
rows.is_empty()
}
None => false,
};
if empty {
self.col.page_rows.remove(&r.vec_page);
self.release_pages(r.vec_page, 1);
}
}
pub(super) fn rerank_for(&self, ann: &IvfPqIndex, req: &RecallRequest) -> Option<usize> {
if req.n_rerank.is_some() || !self.is_v9() {
return req.n_rerank;
}
let (_, auto) = ann.effective_defaults();
Some(auto.max(V9_RERANK_PER_HIT.saturating_mul(req.top_k)))
}
pub(super) fn turn_is_cold(&self, created_at: i64) -> bool {
self.turn_compress_after == 0
|| created_at <= memory::now_secs().saturating_sub(self.turn_compress_after)
}
pub(super) fn unsettled_turn(&self, idx: usize) -> bool {
let e = &self.catalog[idx];
!e.deleted
&& e.v9
.is_some_and(|r| r.kind == ROW_KIND_TURN && r.flags & ROW_SETTLED == 0)
}
pub(super) fn settle_turn(&mut self, idx: usize, bytes: Vec<u8>) -> Vec<u8> {
let out = if bytes.starts_with(&ZSTD_MAGIC) {
bytes
} else {
compress_turn_record(bytes)
};
let e = &mut self.catalog[idx];
e.len = out.len() as u32;
if let Some(r) = e.v9.as_mut() {
r.flags |= ROW_SETTLED;
}
self.touch(idx);
self.dirty_catalog = true;
out
}
pub(super) fn settle_cold_turns(&mut self) -> Result<()> {
if !self.is_v9() || !self.dirty_catalog {
return Ok(());
}
let picked: Vec<usize> = (0..self.catalog.len())
.filter(|&i| {
self.unsettled_turn(i)
&& self.catalog[i]
.v9
.is_some_and(|r| self.turn_is_cold(r.created_at))
})
.take(COLD_TURNS_PER_COMMIT)
.collect();
for idx in picked {
let old = self.catalog[idx].clone();
let bytes = self.read_record(&old)?;
let before = bytes.len();
let out = self.settle_turn(idx, bytes);
if out.len() == before {
continue;
}
self.kill_record(idx, &old);
let gen = self.write_gen();
let e = &mut self.catalog[idx];
if out.len() + REC_SLOT_BYTES <= REC_PAGE_CAP {
e.start_page = 0;
e.page_count = 1;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = 0;
}
self.recs.hot.insert(idx, out);
self.recs.pending.push(idx);
} else {
let (start, pc, gen) = self.write_record(&out)?;
let e = &mut self.catalog[idx];
e.start_page = start;
e.page_count = pc;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = REC_SPAN;
}
}
}
Ok(())
}
pub(super) fn kill_record(&mut self, idx: usize, row: &CatalogEntry) {
self.recs.hot.remove(&idx);
let Some(r) = row.v9 else { return };
if r.rec_slot == REC_SPAN {
self.release_pages(row.start_page, row.page_count as u64);
return;
}
if row.start_page == 0 {
return;
}
let empty = match self.recs.page_rows.get_mut(&row.start_page) {
Some(rows) => {
rows.retain(|&i| i != idx);
rows.is_empty()
}
None => false,
};
if empty {
self.recs.page_rows.remove(&row.start_page);
self.release_pages(row.start_page, 1);
}
}
pub(super) fn pack_records(&mut self) -> Result<()> {
if !self.is_v9() || self.recs.pending.is_empty() {
return Ok(());
}
let mut sparse: Vec<(u64, usize)> = self
.recs
.page_rows
.iter()
.filter(|(_, rows)| {
let live: usize = rows
.iter()
.map(|&i| self.catalog[i].len as usize + REC_SLOT_BYTES)
.sum();
live * 4 < REC_PAGE_CAP * 3
})
.map(|(&p, rows)| (p, rows.len()))
.collect();
sparse.sort_unstable();
let sparse = vacuum_pick(sparse);
let mut order: Vec<usize> = Vec::new();
let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
for p in sparse {
let rows = self.recs.page_rows.remove(&p).unwrap_or_default();
if let Some(&first) = rows.first() {
let page = self.pager.read_page_uncached(p, self.catalog[first].gen)?;
for idx in rows {
let slot = self.catalog[idx].v9.map_or(0, |r| r.rec_slot);
let mut b = record_slot(&page, slot)?;
if self.unsettled_turn(idx) {
b = self.settle_turn(idx, b);
}
self.recs.hot.insert(idx, b);
if seen.insert(idx) {
order.push(idx);
}
}
}
self.release_pages(p, 1);
}
let pending = std::mem::take(&mut self.recs.pending);
for idx in pending {
if self.recs.hot.contains_key(&idx) && seen.insert(idx) {
order.push(idx);
}
}
let gen = self.write_gen();
let mut i = 0;
while i < order.len() {
let mut used = 0usize;
let mut j = i;
while j < order.len() {
let l = self.recs.hot[&order[j]].len() + REC_SLOT_BYTES;
if j > i && used + l > REC_PAGE_CAP {
break;
}
used += l;
j += 1;
}
let page = self.alloc_pages(1);
let bytes = {
let recs: Vec<&[u8]> = order[i..j]
.iter()
.map(|k| self.recs.hot[k].as_slice())
.collect();
encode_record_page(gen, &recs)
};
self.pager.write_page(page, gen, &bytes)?;
for (slot, &k) in order[i..j].iter().enumerate() {
self.touch(k);
let e = &mut self.catalog[k];
e.start_page = page;
e.page_count = 1;
e.gen = gen;
if let Some(r) = e.v9.as_mut() {
r.rec_slot = slot as u16;
}
}
self.recs.page_rows.insert(page, order[i..j].to_vec());
i = j;
}
for k in order {
self.recs.hot.remove(&k);
}
self.dirty_catalog = true;
Ok(())
}
pub(super) fn column_entry(&mut self, e: &CatalogEntry) -> Result<(Vec<u16>, f32)> {
let d = self.dimensions;
if let Some(idx) = self.index.get(e.id, &self.catalog) {
if let Some((h, n)) = self.col.hot.get(&idx) {
return Ok((h.clone(), *n));
}
if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
return Ok((
self.col.data[idx * d..(idx + 1) * d].to_vec(),
self.col.norms[idx],
));
}
}
let r =
e.v9.ok_or_else(|| MnemoError::Invalid("v9 row without a vector entry".into()))?;
if r.vec_page == 0 {
return Err(MnemoError::Invalid("vector entry not written yet".into()));
}
let page = self.pager.read_page(r.vec_page, r.vec_gen)?;
veccol::decode_entry(&page, d, r.vec_slot as usize)
}
pub(super) fn scored_rows(&self) -> usize {
self.catalog.iter().filter(|e| scored_row(e)).count()
}
pub(super) fn recall_mode(&mut self) -> Result<RecallMode> {
self.ensure_ann()?;
if self.ann.is_some() {
return Ok(RecallMode::Indexed);
}
if self.is_v9()
&& self.scored_rows() * veccol::entry_len(self.dimensions) <= self.vector_cache_budget
{
return Ok(RecallMode::ExactResident);
}
Ok(RecallMode::ExactPaged)
}
pub(super) fn ensure_column(&mut self) -> Result<()> {
if !self.is_v9() || self.col.resident {
return Ok(());
}
let d = self.dimensions;
let rows = self.catalog.len();
let scored = self.scored_rows();
if scored * veccol::entry_len(d) > self.vector_cache_budget {
return Ok(());
}
self.col.scored_rows = scored;
self.col.data = vec![0u16; rows * d];
self.col.norms = vec![0f32; rows];
self.col.have = vec![false; rows];
let hot: Vec<(usize, Vec<u16>, f32)> = self
.col
.hot
.iter()
.filter(|(&i, _)| i < rows)
.map(|(&i, (h, n))| (i, h.clone(), *n))
.collect();
for (i, h, n) in hot {
self.col.data[i * d..(i + 1) * d].copy_from_slice(&h);
self.col.norms[i] = n;
self.col.have[i] = true;
}
self.col.resident = true;
Ok(())
}
pub(super) fn fill_column_page(&mut self, idx: usize) -> Result<()> {
let Some(r) = self.catalog[idx].v9 else {
return Ok(());
};
if r.vec_page == 0 {
return Ok(());
}
let d = self.dimensions;
let bytes = self.pager.read_page_uncached(r.vec_page, r.vec_gen)?;
let rows = self
.col
.page_rows
.get(&r.vec_page)
.cloned()
.unwrap_or_default();
for i in rows.into_iter().chain(std::iter::once(idx)) {
if i >= self.col.have.len() || self.col.have[i] {
continue;
}
if i != idx && !scored_row(&self.catalog[i]) {
continue;
}
let Some(ri) = self.catalog[i].v9 else {
continue;
};
if ri.vec_page != r.vec_page {
continue;
}
let n = veccol::decode_entry_into(
&bytes,
d,
ri.vec_slot as usize,
&mut self.col.data[i * d..(i + 1) * d],
)?;
self.col.norms[i] = n;
self.col.have[i] = true;
}
Ok(())
}
pub(super) fn pack_vectors(&mut self) -> Result<()> {
if !self.is_v9() || self.col.pending.is_empty() {
return Ok(());
}
let d = self.dimensions;
let per = veccol::per_page(d);
let mut sparse: Vec<(u64, usize)> = self
.col
.page_rows
.iter()
.filter(|(_, rows)| rows.len() * 4 < per * 3)
.map(|(&p, rows)| (p, rows.len()))
.collect();
sparse.sort_unstable();
let sparse = vacuum_pick(sparse);
let mut order: Vec<usize> = Vec::new();
let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
for p in sparse {
let rows = self.col.page_rows.remove(&p).unwrap_or_default();
for idx in rows {
let e = self.catalog[idx].clone();
let v = self.column_entry(&e)?;
self.col.hot.insert(idx, v);
if seen.insert(idx) {
order.push(idx);
}
}
self.release_pages(p, 1);
}
let pending = std::mem::take(&mut self.col.pending);
for idx in pending {
if self.col.hot.contains_key(&idx) && seen.insert(idx) {
order.push(idx);
}
}
let gen = self.write_gen();
for chunk in order.chunks(per) {
let page = self.alloc_pages(1);
let bytes = {
let entries: Vec<(&[u16], f32)> = chunk
.iter()
.map(|i| {
let (h, n) = &self.col.hot[i];
(h.as_slice(), *n)
})
.collect();
veccol::encode_page(d, gen, &entries)
};
self.pager.write_page(page, gen, &bytes)?;
for (slot, &i) in chunk.iter().enumerate() {
self.touch(i);
if let Some(r) = self.catalog[i].v9.as_mut() {
r.vec_page = page;
r.vec_slot = slot as u16;
r.vec_gen = gen;
}
}
self.col.page_rows.insert(page, chunk.to_vec());
}
for i in order {
self.col.hot.remove(&i);
}
self.dirty_catalog = true;
Ok(())
}
pub(super) fn ensure_gen_lease(&mut self) -> Result<()> {
if !self.is_v9() || self.gen_leased || self.read_only {
return Ok(());
}
self.header.wal_seq += 1;
let wal_start = self.header.wal_start;
self.relocate_header_through_wal(wal_start)?;
self.gen_leased = true;
Ok(())
}
pub(super) fn write_record(&mut self, bytes: &[u8]) -> Result<(u64, u32, u64)> {
self.ensure_gen_lease()?;
let pc = bytes.len().div_ceil(PAYLOAD).max(1);
let start = self.alloc_pages(pc as u64);
let gen = self.write_gen();
for i in 0..pc {
let lo = i * PAYLOAD;
let hi = ((i + 1) * PAYLOAD).min(bytes.len());
self.pager
.write_page(start + i as u64, gen, &bytes[lo..hi])?;
}
Ok((start, pc as u32, gen))
}
pub(super) fn read_record_run(
&mut self,
start: u64,
pages: u32,
len: u32,
gen: u64,
) -> Result<Vec<u8>> {
let mut buf = Vec::with_capacity(len as usize);
for i in 0..pages as u64 {
buf.extend_from_slice(&self.pager.read_page(start + i, gen)?);
}
buf.truncate(len as usize);
Ok(buf)
}
pub(super) fn read_record(&mut self, e: &CatalogEntry) -> Result<Vec<u8>> {
if let Some(r) = e.v9 {
if r.rec_slot != REC_SPAN {
if e.start_page == 0 {
return self
.index
.get(e.id, &self.catalog)
.and_then(|idx| self.recs.hot.get(&idx).cloned())
.ok_or_else(|| MnemoError::Invalid("record not written yet".into()));
}
let page = self.pager.read_page(e.start_page, e.gen)?;
return record_slot(&page, r.rec_slot);
}
}
let mut buf = Vec::with_capacity(e.len as usize);
for i in 0..e.page_count as u64 {
buf.extend_from_slice(&self.pager.read_page(e.start_page + i, e.gen)?);
}
buf.truncate(e.len as usize);
Ok(buf)
}
pub(super) fn decode_memory(&self, bytes: &[u8]) -> Result<Memory> {
let plain = if self.is_v9() {
plain_record(bytes)?
} else {
std::borrow::Cow::Borrowed(bytes)
};
rmp_serde::from_slice(&plain).map_err(|err| MnemoError::Serialize(err.to_string()))
}
pub(super) fn read_memory(&mut self, e: &CatalogEntry) -> Result<Memory> {
let m = self.read_memory_meta(e)?;
self.finish_memory(e, m)
}
pub(super) fn finish_memory(&mut self, e: &CatalogEntry, mut m: Memory) -> Result<Memory> {
if e.v9.is_some() && !self.float_vectors() {
let (h, _) = self.column_entry(e)?;
m.vector = veccol::dequantize(&h);
}
m.accessed_at = e.accessed_at;
m.access_count = e.access_count;
Ok(m)
}
pub(super) fn put(&mut self, m: Memory) -> Result<Ulid> {
self.put_checked(m, true)
}
pub(super) fn put_existing(&mut self, m: Memory) -> Result<Ulid> {
self.put_checked(m, false)
}
fn put_checked(&mut self, mut m: Memory, refuse_zero: bool) -> Result<Ulid> {
if m.vector.len() != self.dimensions {
return Err(MnemoError::DimensionMismatch {
expected: self.dimensions,
got: m.vector.len(),
});
}
if refuse_zero && embed::is_zero_vector(&m.vector) {
return Err(MnemoError::Invalid(
"all-zero vector refused: it scores 0.0 on every cosine query and \
pollutes recall; check your embedder returned a real embedding"
.into(),
));
}
if m.id == Ulid::nil() {
m.id = Ulid::new();
}
let id_u: u128 = m.id.0;
if self.layers.is_some() {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
let old_entry = self.catalog[idx].clone();
if !old_entry.deleted {
let old = self.read_memory(&old_entry)?;
self.note_removed(&old)?;
}
}
}
let v9 = self.is_v9();
let bytes = if v9 && !self.float_vectors() {
let vector = std::mem::take(&mut m.vector);
let b = rmp_serde::to_vec(&m);
m.vector = vector;
b
} else {
rmp_serde::to_vec(&m)
}
.map_err(|e| MnemoError::Serialize(e.to_string()))?;
let is_turn = v9 && meta_str(&m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN);
let settle =
is_turn && (bytes.len() < TURN_ZSTD_MIN_BYTES || self.turn_is_cold(m.created_at));
let bytes = if settle {
compress_turn_record(bytes)
} else {
bytes
};
let packed = v9 && bytes.len() + REC_SLOT_BYTES <= REC_PAGE_CAP;
let (start, pc, gen) = if packed {
self.ensure_gen_lease()?;
(0, 1, self.write_gen())
} else {
self.write_record(&bytes)?
};
let rec_len = bytes.len();
let mut tags = v9.then(|| row_tags(&m));
if let Some(r) = tags.as_mut() {
if bm25_kind(r.kind) {
let (terms, len) = text::term_freqs(&m.content);
r.doc = self.bm25.manifest.next_doc;
r.doc_len = len;
self.bm25.manifest.next_doc += 1;
self.bm25.pending.push((r.doc, terms));
}
}
let bm25_new = tags.filter(|r| r.doc != 0 && bm25_kind(r.kind));
let (prev_accessed, prev_count) = match self.index.get(id_u, &self.catalog) {
Some(idx) => (
self.catalog[idx].accessed_at,
self.catalog[idx].access_count,
),
None => (m.accessed_at, m.access_count),
};
let entry = CatalogEntry {
id: id_u,
start_page: start,
page_count: pc,
len: rec_len as u32,
deleted: false,
accessed_at: prev_accessed,
access_count: prev_count,
gen,
v9: tags.map(|mut r| {
r.rec_slot = if packed { 0 } else { REC_SPAN };
if settle {
r.flags |= ROW_SETTLED;
}
r
}),
};
match self.index.get(id_u, &self.catalog) {
Some(idx) => {
if !self.catalog[idx].deleted {
let old = self.catalog[idx].clone();
if v9 {
self.kill_vector(idx, &old);
self.kill_record(idx, &old);
} else {
self.release_pages(old.start_page, old.page_count as u64);
}
}
if let (Some(stats), Some(old)) = (self.bm25.stats.as_mut(), self.catalog[idx].v9) {
if !self.catalog[idx].deleted && old.doc != 0 {
stats.remove(old.doc);
}
}
self.catalog[idx] = entry;
self.touch(idx);
}
None => {
self.index.insert(id_u, self.catalog.len(), &self.catalog);
self.touch(self.catalog.len());
self.catalog.push(entry);
}
}
if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), bm25_new) {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
stats.insert(r.doc, idx, r.doc_len);
}
}
self.dirty_catalog = true;
if v9 {
if let Some(idx) = self.index.get(id_u, &self.catalog) {
self.stage_vector(idx, &m.vector);
if packed {
self.recs.hot.insert(idx, bytes);
self.recs.pending.push(idx);
}
}
}
if let Some(ann) = &mut self.ann {
if is_ann_kind(&m) {
ann.add(id_u, &m.vector);
self.dirty_index = true;
self.index_persist.changes += 1;
} else if ann.contains(id_u) {
ann.remove(id_u);
self.dirty_index = true;
self.index_persist.changes += 1;
}
}
self.note_added(&m)?;
if let Some(set) = &mut self.agent_set {
set.insert(m.agent_id.clone());
}
Ok(m.id)
}
}