use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use fs4::fs_std::FileExt;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use ulid::Ulid;
use crate::crypto::{self, KdfParams};
use crate::embed::{self, Embedder};
use crate::error::{MnemoError, Result};
use crate::format::{
Header, RunPtr, FLAG_ENCRYPTED, PAGE_SIZE, PAYLOAD, VERSION, VERSION_V9, WRAPPED_DEK_LEN,
};
use crate::freelist::FreeList;
use crate::index::{IndexConfig, IndexInfo, IndexOp, IvfPqIndex};
#[cfg(feature = "dev-hooks")]
use crate::keyslot::KeySlot as KeySlotEntry;
use crate::keyslot::KeySlotRun;
use crate::memory::{
self, status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind,
EssenceScope, EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, Memory,
MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_TRANSITIONS,
MNEMO_META_CONTINUES, MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND,
MNEMO_META_KIND_ESSENCE, MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH,
MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
};
use crate::pager::{LockedFile, Pager};
use crate::result_cache::{
self, BatchState, CacheBudget, CacheDirectoryEntry, CacheEntry, CacheFlushPolicy, CachePutOpts,
CacheStats, CachedValue, CounterState, DirectoryIndex, SemanticCachePutOpts,
};
use crate::text;
use crate::veccol;
use crate::wal;
mod ann;
mod cache_api;
mod continuation;
mod crud;
mod essence;
mod flush;
#[cfg(test)]
mod header_graft_tests;
#[cfg(feature = "dev-hooks")]
mod hooks;
mod keyed;
mod load;
mod maintenance;
mod open;
mod recall;
mod records;
mod snapshots;
mod threads;
use crud::*;
use essence::*;
use load::*;
pub(crate) use records::plain_record;
use records::*;
const DEFAULT_WAL_PAGES: u64 = 8;
const MIN_WAL_PAGES: u64 = 2;
pub const DEFAULT_MAX_SNAPSHOTS: usize = 256;
pub const INDEX_PERSIST_EVERY_COMMITS: u32 = 64;
pub const INDEX_PERSIST_EVERY_CHANGES: usize = 1_000;
pub const DEFAULT_KEYWORD_WEIGHT: f32 = 0.5;
pub const DEFAULT_VECTOR_CACHE_BUDGET: usize = 96 * 1024 * 1024;
pub const V9_ANN_MIN_FACTS: usize = 20_000;
pub const V9_RERANK_PER_HIT: usize = 50;
const V9_WAL_RUN_MAX_PAGES: u64 = 2;
pub const V9_DEFAULT_MAX_SNAPSHOTS: usize = 16;
const V9_SNAPSHOT_KEEP_SECS: i64 = 24 * 3600;
const V9_PIN_FLOOR_PAGES: u64 = 128;
const V9_INDEX_LAYOUT: u8 = 1;
const VACUUM_PAGES_PER_COMMIT: usize = 4;
const VACUUM_MAX_ROWS: usize = 64;
const FLAG_FLOAT_VECTORS: u32 = 2;
#[derive(Debug, Default, Clone, Copy)]
struct IndexPersist {
forced: bool,
commits: u32,
changes: usize,
}
#[derive(Clone, Copy, Debug)]
pub struct MnemoConfig {
pub dimensions: usize,
pub kdf: KdfParams,
pub wal_pages_initial: u64,
pub max_snapshots: usize,
#[doc(hidden)]
pub format_version: u16,
pub store_float_vectors: bool,
}
impl Default for MnemoConfig {
fn default() -> Self {
Self {
dimensions: 768,
kdf: KdfParams::secure(),
wal_pages_initial: DEFAULT_WAL_PAGES,
max_snapshots: DEFAULT_MAX_SNAPSHOTS,
format_version: 0,
store_float_vectors: false,
}
}
}
fn resolve_format_version(requested: u16) -> Result<u16> {
match requested {
0 => Ok(crate::format::DEFAULT_NEW_VERSION),
VERSION_V9 => Ok(requested),
v => Err(MnemoError::Invalid(format!(
"format_version {v} is not supported for new files: new files are v9 (pass 0 or 9)"
))),
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct CatalogEntry {
id: u128,
start_page: u64,
page_count: u32,
len: u32,
deleted: bool,
accessed_at: i64,
access_count: u32,
#[serde(default, skip_serializing_if = "crate::format::is_zero_u64")]
gen: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
v9: Option<RowV9>,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
struct RowV9 {
vec_page: u64,
vec_slot: u16,
vec_gen: u64,
rec_slot: u16,
kind: u8,
flags: u8,
memory_type: u8,
thread_hash: u32,
key_hash: u32,
agent_hash: u32,
created_at: i64,
importance: f32,
expires_at: i64,
doc: u32,
doc_len: u16,
}
pub const DEFAULT_TURN_COMPRESS_AFTER_SECS: i64 = 7 * 24 * 3600;
#[derive(Serialize, Deserialize, Clone, Debug)]
struct CatalogEntryV4 {
id: u128,
start_page: u64,
page_count: u32,
len: u32,
deleted: bool,
}
struct FlushPrelude {
cat_bytes: Option<Vec<u8>>,
cat_delta: bool,
idx_delta: bool,
bm25_seg: Option<(Vec<u8>, Option<usize>)>,
idx_bytes: Option<Vec<u8>>,
cache_bytes: Option<Vec<u8>>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct Snapshot {
txn_id: u64,
created_at: i64,
catalog_start: u64,
catalog_pages: u64,
catalog_len: u64,
index_start: u64,
index_pages: u64,
index_len: u64,
memory_count: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
gens: Option<SnapshotGens>,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
struct SnapshotGens {
catalog: u64,
index: u64,
#[serde(default)]
delta: (u64, u64, u64, u64),
#[serde(default)]
index_delta: (u64, u64, u64, u64),
#[serde(default)]
bm25: (u64, u64, u64, u64),
}
impl Snapshot {
fn catalog_gen(&self) -> u64 {
self.gens.map_or(0, |g| g.catalog)
}
fn index_gen(&self) -> u64 {
self.gens.map_or(0, |g| g.index)
}
fn delta_head(&self) -> RunPtr {
self.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.delta))
}
fn index_delta_head(&self) -> RunPtr {
self.gens
.map_or(RunPtr::default(), |g| tuple_ptr(g.index_delta))
}
}
#[derive(Clone, Copy, Debug)]
pub struct SnapshotInfo {
pub txn_id: u64,
pub created_at: i64,
pub memory_count: u64,
}
pub(super) struct RestorePoints {
pub(super) retained: usize,
pub(super) oldest_at: Option<i64>,
pub(super) pinned_bytes: u64,
}
#[derive(Clone, Debug)]
pub struct RecallRequest {
pub query: Vec<f32>,
pub top_k: usize,
pub memory_types: Option<Vec<MemoryType>>,
pub agent_id: Option<String>,
pub metric: Metric,
pub weights: ScoreWeights,
pub n_probe: Option<usize>,
pub n_rerank: Option<usize>,
pub track_access: bool,
pub filter: Filter,
pub max_bytes: Option<usize>,
pub keywords: Option<String>,
pub keyword_weight: f32,
pub bm25_k1: f32,
pub bm25_b: f32,
}
impl RecallRequest {
pub fn new(query: Vec<f32>) -> Self {
Self {
query,
top_k: 10,
memory_types: None,
agent_id: None,
metric: Metric::Cosine,
weights: ScoreWeights::default(),
n_probe: None,
n_rerank: None,
track_access: true,
filter: Filter::default(),
max_bytes: None,
keywords: None,
keyword_weight: DEFAULT_KEYWORD_WEIGHT,
bm25_k1: 1.2,
bm25_b: 0.75,
}
}
pub fn keywords_only(keywords: impl Into<String>) -> Self {
Self::new(Vec::new()).keywords(keywords)
}
pub fn keywords(mut self, text: impl Into<String>) -> Self {
self.keywords = Some(text.into());
self
}
pub fn keyword_weight(mut self, w: f32) -> Self {
self.keyword_weight = w;
self
}
pub fn filter(mut self, filter: Filter) -> Self {
self.filter = filter;
self
}
pub fn max_bytes(mut self, n: usize) -> Self {
self.max_bytes = Some(n);
self
}
pub fn from_text(embedder: &dyn Embedder, query_text: &str) -> Result<Self> {
let mut vectors = embedder.embed(&[query_text])?;
let v = vectors.pop().ok_or_else(|| {
MnemoError::Invalid("embedder returned no vector for the query text".into())
})?;
Ok(Self::new(v))
}
pub fn top_k(mut self, k: usize) -> Self {
self.top_k = k;
self
}
pub fn types(mut self, t: Vec<MemoryType>) -> Self {
self.memory_types = Some(t);
self
}
pub fn agent(mut self, agent_id: impl Into<String>) -> Self {
self.agent_id = Some(agent_id.into());
self
}
pub fn metric(mut self, metric: Metric) -> Self {
self.metric = metric;
self
}
pub fn weights(mut self, weights: ScoreWeights) -> Self {
self.weights = weights;
self
}
pub fn n_probe(mut self, n: usize) -> Self {
self.n_probe = Some(n);
self
}
pub fn n_rerank(mut self, n: usize) -> Self {
self.n_rerank = Some(n);
self
}
pub fn track_access(mut self, track: bool) -> Self {
self.track_access = track;
self
}
}
#[derive(Clone, Debug)]
pub struct RecallResult {
pub memory: Memory,
pub score: f32,
pub similarity: f32,
}
#[derive(Clone, Debug)]
pub struct RecallResponse {
pub hits: Vec<RecallResult>,
pub truncated: bool,
pub used_exact_scan: bool,
}
#[doc(hidden)]
#[derive(Clone, Debug, Default, serde::Serialize)]
#[cfg(feature = "dev-hooks")]
pub struct BudgetReport {
pub file_bytes: u64,
pub page_size: u64,
pub header_bytes: u64,
pub record_page_bytes: u64,
pub record_payload_bytes: u64,
pub deleted_record_page_bytes: u64,
pub catalog_run_bytes: u64,
pub catalog_payload_bytes: u64,
pub index_run_bytes: u64,
pub index_payload_bytes: u64,
pub index_start_page: u64,
pub wal_bytes: u64,
pub snapshot_manifest_bytes: u64,
pub cache_dir_run_bytes: u64,
pub unreferenced_bytes: u64,
pub live_records: u64,
pub ram_catalog: u64,
pub ram_id_map: u64,
pub ram_ann_centroids: u64,
pub ram_ann_codebook: u64,
pub ram_ann_posting_ids: u64,
pub ram_ann_posting_codes: u64,
pub ram_ann_assignment: u64,
pub ram_cache_dir: u64,
pub ram_key_caches: u64,
pub ram_vector_column: u64,
pub ram_total: u64,
}
#[derive(Clone, Debug)]
pub struct Stats {
pub memories: usize,
pub deleted: usize,
pub dimensions: usize,
pub file_bytes: u64,
pub agents: Vec<String>,
pub encrypted: bool,
pub created_at: i64,
pub index: Option<IndexInfo>,
pub wal_pages: u64,
pub fsyncs_last_flush: u32,
pub format_version: u16,
pub recall_mode: RecallMode,
pub snapshots_retained: usize,
pub oldest_restorable_at: Option<i64>,
pub snapshot_pinned_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StatsSummary {
pub memories: usize,
pub deleted: usize,
pub dimensions: usize,
pub file_bytes: u64,
pub format_version: u16,
pub wal_pages: u64,
pub has_index: bool,
pub recall_mode: RecallMode,
pub vector_column_bytes: u64,
pub snapshots_retained: usize,
pub oldest_restorable_at: Option<i64>,
pub snapshot_pinned_bytes: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RecallMode {
ExactResident,
ExactPaged,
Indexed,
}
impl RecallMode {
pub fn as_str(&self) -> &'static str {
match self {
RecallMode::ExactResident => "exact_resident",
RecallMode::ExactPaged => "exact_paged",
RecallMode::Indexed => "indexed",
}
}
}
pub struct CloseError {
pub(super) handle: Box<Mnemo>,
pub(super) error: MnemoError,
}
impl CloseError {
pub fn error(&self) -> &MnemoError {
&self.error
}
pub fn into_inner(self) -> Mnemo {
*self.handle
}
pub fn into_parts(self) -> (Mnemo, MnemoError) {
(*self.handle, self.error)
}
}
impl std::fmt::Debug for CloseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CloseError")
.field("error", &self.error)
.finish_non_exhaustive()
}
}
impl std::fmt::Display for CloseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "close failed (the handle is kept): {}", self.error)
}
}
impl std::error::Error for CloseError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct CompactReport {
pub before: usize,
pub after: usize,
pub cache_before: usize,
pub cache_after: usize,
pub zero_vector_manifests: usize,
pub zero_vector_memories: usize,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VerifyReport {
pub records: usize,
pub zero_vector_memories: usize,
}
pub struct Mnemo {
pager: Pager,
header: Header,
catalog: Vec<CatalogEntry>,
index: IdIndex,
#[allow(dead_code)]
path: PathBuf,
dimensions: usize,
kdf: KdfParams,
dirty_catalog: bool,
ann: Option<IvfPqIndex>,
dirty_index: bool,
index_persist: IndexPersist,
ann_unloaded: bool,
manifest: Vec<Snapshot>,
max_snapshots: usize,
read_only: bool,
cache_dir: Vec<CacheDirectoryEntry>,
cache_idx: DirectoryIndex,
dirty_cache: bool,
cache_budgets: std::collections::HashMap<String, CacheBudget>,
cache_flush_policy: CacheFlushPolicy,
cache_batch: BatchState,
cache_counters: CounterState,
embedder: Option<std::sync::Arc<dyn Embedder>>,
fsyncs_last_flush: u32,
agent_set: Option<std::collections::BTreeSet<String>>,
key_index: Option<std::collections::HashMap<KeySlot, u128>>,
superseded: Option<std::collections::HashSet<u128>>,
layers: Option<LayerCache>,
turn_compress_after: i64,
free: FreeList,
pending_free: Vec<(u64, u64)>,
free_dirty: bool,
key_slots: KeySlotRun,
dirty_keys: bool,
gen_leased: bool,
col: VecColumn,
recs: RecPages,
bm25: Bm25State,
index_layout_override: Option<u8>,
kw: Option<std::collections::HashMap<u128, f32>>,
dirty_rows: std::collections::BTreeSet<usize>,
delta_chain: Vec<RunPtr>,
delta_rows: usize,
index_chain: Vec<RunPtr>,
index_chain_ops: usize,
index_rebase: bool,
index_delta_ops: Option<Vec<IndexOp>>,
catalog_rewrite: bool,
snapshot_cap_explicit: bool,
direct_writes: bool,
vector_cache_budget: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) struct KeySlot {
pub(crate) scope_owner: Option<String>,
pub(crate) key: String,
}
fn is_manifest(m: &Memory) -> bool {
m.metadata
.get("topic")
.and_then(|v| v.as_str())
.map(|s| s.eq_ignore_ascii_case("manifest"))
.unwrap_or(false)
}
#[derive(Debug, Clone, Default)]
struct IdIndex {
slots: Vec<u32>,
len: usize,
}
const NO_ROW: u32 = u32::MAX;
impl IdIndex {
fn build(catalog: &[CatalogEntry]) -> IdIndex {
let mut t = IdIndex {
slots: vec![NO_ROW; crate::index::table_capacity_for(catalog.len())],
len: 0,
};
for (row, e) in catalog.iter().enumerate() {
t.insert(e.id, row, catalog);
}
t
}
fn get(&self, id: u128, catalog: &[CatalogEntry]) -> Option<usize> {
if self.slots.is_empty() {
return None;
}
let cap = self.slots.len();
let mut i = crate::index::slot_home(id, cap);
loop {
let row = self.slots[i];
if row == NO_ROW {
return None;
}
if catalog[row as usize].id == id {
return Some(row as usize);
}
i = crate::index::next_slot(i, cap);
}
}
fn insert(&mut self, id: u128, row: usize, catalog: &[CatalogEntry]) {
if (self.len + 1) * 4 > self.slots.len() * 3 {
let cap = (self.slots.len() * 2).max(crate::index::table_capacity_for(self.len + 1));
let old = std::mem::replace(&mut self.slots, vec![NO_ROW; cap]);
self.len = 0;
for r in old.into_iter().filter(|&r| r != NO_ROW) {
self.insert(catalog[r as usize].id, r as usize, catalog);
}
}
let cap = self.slots.len();
let mut i = crate::index::slot_home(id, cap);
loop {
let r = self.slots[i];
if r == NO_ROW {
self.slots[i] = row as u32;
self.len += 1;
return;
}
if catalog[r as usize].id == id {
self.slots[i] = row as u32;
return;
}
i = crate::index::next_slot(i, cap);
}
}
#[cfg(feature = "dev-hooks")]
fn heap_bytes(&self) -> usize {
self.slots.capacity() * std::mem::size_of::<u32>()
}
}
fn build_id_index(catalog: &[CatalogEntry]) -> IdIndex {
IdIndex::build(catalog)
}
fn written_after_index(e: &CatalogEntry, v9: bool, index_start: u64, index_gen: u64) -> bool {
if v9 {
e.start_page == 0 || e.gen > index_gen
} else {
e.start_page > index_start
}
}
fn is_ann_kind(m: &Memory) -> bool {
!matches!(
m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
Some(MNEMO_META_KIND_TURN)
| Some(MNEMO_META_KIND_ESSENCE)
| Some(MNEMO_META_KIND_THREAD_LINK)
)
}