pub mod bm25;
pub mod embedder;
pub use bm25::{Bm25Index, Bm25Params, reciprocal_rank_fusion};
pub use embedder::{
DeterministicHashEmbedder, EmbeddingEngine, EmbeddingProvider, HttpEmbeddingEngine,
};
use crate::error::{Error, Result};
pub mod api;
pub(crate) mod binary;
mod config;
mod indexed;
pub use api::*;
pub use config::*;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MemoryEntry {
pub id: u64,
pub content: String,
pub vector: Option<Vec<f32>>,
pub timestamp: u64,
pub last_accessed: u64,
pub importance: f32,
pub access_count: u32,
pub tags: Vec<String>,
pub associations: Vec<u64>,
#[serde(default)]
pub namespace: Option<String>,
#[serde(default)]
pub session_id: Option<String>,
#[serde(default)]
pub external_id: Option<String>,
#[serde(default)]
pub source_version: Option<String>,
#[serde(default)]
pub embedding_model: Option<String>,
#[serde(default)]
pub expires_at: Option<u64>,
#[serde(default)]
pub revision: u64,
#[serde(default)]
pub updated_at: u64,
#[serde(default)]
pub content_hash: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecallFilter {
pub vector_weight: f32,
pub bm25_weight: f32,
pub recency_weight: f32,
pub importance_weight: f32,
pub half_life_seconds: u64,
pub required_tags: Vec<String>,
pub min_score: f32,
pub graph_expansion_hops: usize,
#[serde(default)]
pub namespace: Option<String>,
#[serde(default)]
pub session_id: Option<String>,
}
impl Default for MemoryRecallFilter {
fn default() -> Self {
Self {
vector_weight: 0.45,
bm25_weight: 0.35,
recency_weight: 0.10,
importance_weight: 0.10,
half_life_seconds: 30 * 86400, required_tags: Vec::new(),
min_score: 0.0,
graph_expansion_hops: 0,
namespace: None,
session_id: None,
}
}
}
impl MemoryRecallFilter {
pub fn with_weights(mut self, vector: f32, bm25: f32, recency: f32, importance: f32) -> Self {
self.vector_weight = vector;
self.bm25_weight = bm25;
self.recency_weight = recency;
self.importance_weight = importance;
self
}
pub fn with_half_life(mut self, seconds: u64) -> Self {
self.half_life_seconds = seconds;
self
}
pub fn with_tags(mut self, tags: &[&str]) -> Self {
self.required_tags = tags.iter().map(|t| t.to_string()).collect();
self
}
pub fn with_graph_hops(mut self, hops: usize) -> Self {
self.graph_expansion_hops = hops;
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecallResult {
pub entry: MemoryEntry,
pub combined_score: f32,
pub semantic_score: f32,
pub lexical_score: f32,
pub recency_score: f32,
pub is_associative: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryEngine {
entries: HashMap<u64, MemoryEntry>,
bm25: Bm25Index,
next_id: u64,
#[serde(skip)]
runtime: indexed::MemoryRuntime,
}
impl Default for MemoryEngine {
fn default() -> Self {
Self::new()
}
}
impl MemoryEngine {
pub fn new() -> Self {
Self {
entries: HashMap::new(),
bm25: Bm25Index::new(),
next_id: 1,
runtime: indexed::MemoryRuntime::default(),
}
}
#[allow(
clippy::too_many_arguments,
reason = "Preserve the public scoped-memory API"
)]
pub fn remember_scoped(
&mut self,
content: &str,
vector: Option<&[f32]>,
importance: f32,
tags: &[&str],
now_ts: u64,
namespace: Option<&str>,
session_id: Option<&str>,
) -> u64 {
let id = self.next_id;
self.next_id += 1;
let entry = MemoryEntry {
id,
content: content.to_string(),
vector: vector.map(|v| v.to_vec()),
timestamp: now_ts,
last_accessed: now_ts,
importance: importance.clamp(0.0, 1.0),
access_count: 0,
tags: tags.iter().map(|s| s.to_string()).collect(),
associations: Vec::new(),
namespace: namespace.map(|s| s.to_string()),
session_id: session_id.map(|s| s.to_string()),
external_id: None,
source_version: None,
embedding_model: None,
expires_at: None,
revision: 1,
updated_at: now_ts,
content_hash: indexed::content_hash(content),
};
self.bm25.index_document(id, content);
self.entries.insert(id, entry);
self.runtime.changed(id, self.entries.get(&id));
id
}
pub fn restore_entry(&mut self, mut entry: MemoryEntry) -> Result<()> {
if entry.id == 0 || entry.id == u64::MAX {
return Err(Error::Corrupted("Invalid persisted memory ID".into()));
}
if entry.content_hash.is_empty() {
entry.content_hash = indexed::content_hash(&entry.content);
}
if entry.revision == 0 {
entry.revision = 1;
}
if entry.updated_at == 0 {
entry.updated_at = entry.timestamp;
}
self.validate_entry(&entry)?;
if entry.id >= self.next_id {
self.next_id = entry.id + 1;
}
self.bm25.index_document(entry.id, &entry.content);
self.runtime.changed(entry.id, Some(&entry));
self.entries.insert(entry.id, entry);
Ok(())
}
pub fn remember(
&mut self,
content: &str,
vector: Option<&[f32]>,
importance: f32,
tags: &[&str],
now_ts: u64,
) -> u64 {
self.remember_scoped(content, vector, importance, tags, now_ts, None, None)
}
pub fn associate(&mut self, id_a: u64, id_b: u64) -> Result<()> {
if !self.entries.contains_key(&id_a) {
return Err(Error::Corrupted(format!("Memory #{id_a} not found")));
}
if !self.entries.contains_key(&id_b) {
return Err(Error::Corrupted(format!("Memory #{id_b} not found")));
}
if let Some(entry_a) = self.entries.get_mut(&id_a)
&& !entry_a.associations.contains(&id_b)
{
entry_a.associations.push(id_b);
}
if let Some(entry_b) = self.entries.get_mut(&id_b)
&& !entry_b.associations.contains(&id_a)
{
entry_b.associations.push(id_a);
}
Ok(())
}
pub fn get_memory_mut(&mut self, id: u64, now_ts: u64) -> Option<&mut MemoryEntry> {
if let Some(entry) = self.entries.get_mut(&id) {
if entry.expires_at.is_some_and(|expiry| now_ts >= expiry) {
return None;
}
entry.last_accessed = now_ts;
entry.access_count += 1;
Some(entry)
} else {
None
}
}
pub fn get_memory(&self, id: u64) -> Option<&MemoryEntry> {
self.entries.get(&id)
}
pub fn memory_count(&self) -> usize {
self.entries.len()
}
pub fn recall(
&mut self,
query_text: Option<&str>,
query_vector: Option<&[f32]>,
limit: usize,
filter: &MemoryRecallFilter,
now_ts: u64,
) -> Vec<MemoryRecallResult> {
if self.entries.is_empty() || limit == 0 {
return Vec::new();
}
let bm25_scores = if let Some(q_text) = query_text {
let matches = self.bm25.search(q_text, self.entries.len());
matches.into_iter().collect::<HashMap<u64, f32>>()
} else {
HashMap::new()
};
let lambda = if filter.half_life_seconds > 0 {
2.0f32.ln() / (filter.half_life_seconds as f32)
} else {
0.0
};
let mut scored_candidates: Vec<(u64, f32, f32, f32, f32, bool)> =
Vec::with_capacity(self.entries.len());
for entry in self.entries.values() {
if entry.expires_at.is_some_and(|expiry| now_ts >= expiry) {
continue;
}
if let Some(ref req_ns) = filter.namespace {
match &entry.namespace {
Some(ns) if ns.eq_ignore_ascii_case(req_ns) => {}
_ => continue,
}
}
if let Some(ref req_sess) = filter.session_id {
match &entry.session_id {
Some(sess) if sess == req_sess => {}
_ => continue,
}
}
if !filter.required_tags.is_empty() {
let matches_tag = filter
.required_tags
.iter()
.any(|req_tag| entry.tags.iter().any(|t| t.eq_ignore_ascii_case(req_tag)));
if !matches_tag {
continue;
}
}
let semantic_score = match (query_vector, &entry.vector) {
(Some(q_vec), Some(e_vec)) => cosine_similarity(q_vec, e_vec).max(0.0),
_ => 0.0,
};
let lexical_score = *bm25_scores.get(&entry.id).unwrap_or(&0.0);
let dt = now_ts.saturating_sub(entry.timestamp) as f32;
let recency_score = (-lambda * dt).exp().clamp(0.0, 1.0);
let combined_score = filter.vector_weight * semantic_score
+ filter.bm25_weight * lexical_score
+ filter.recency_weight * recency_score
+ filter.importance_weight * entry.importance;
let has_query = query_text.is_some() || query_vector.is_some();
let matches_query = if has_query {
(query_text.is_some() && lexical_score > 0.0)
|| (query_vector.is_some() && semantic_score > 0.0)
} else {
true
};
if matches_query && combined_score >= filter.min_score {
scored_candidates.push((
entry.id,
combined_score,
semantic_score,
lexical_score,
recency_score,
false, ));
}
}
scored_candidates
.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
if filter.graph_expansion_hops > 0 && !scored_candidates.is_empty() {
let mut existing_ids: HashSet<u64> = scored_candidates.iter().map(|c| c.0).collect();
let mut associative_candidates = Vec::new();
let base_count = scored_candidates.len().min(limit);
for &(parent_id, parent_score, ..) in scored_candidates.iter().take(base_count) {
if let Some(parent_entry) = self.entries.get(&parent_id) {
for &assoc_id in &parent_entry.associations {
if !existing_ids.contains(&assoc_id)
&& let Some(assoc_entry) = self.entries.get(&assoc_id)
{
if assoc_entry
.expires_at
.is_some_and(|expiry| now_ts >= expiry)
{
continue;
}
if let Some(ref req_ns) = filter.namespace {
match &assoc_entry.namespace {
Some(ns) if ns.eq_ignore_ascii_case(req_ns) => {}
_ => continue,
}
}
if let Some(ref req_sess) = filter.session_id {
match &assoc_entry.session_id {
Some(sess) if sess == req_sess => {}
_ => continue,
}
}
existing_ids.insert(assoc_id);
let assoc_score = parent_score * 0.85;
associative_candidates.push((
assoc_id,
assoc_score,
0.0,
0.0,
0.0,
true,
));
}
}
}
}
scored_candidates.extend(associative_candidates);
scored_candidates
.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
}
scored_candidates.truncate(limit);
let mut final_results = Vec::with_capacity(scored_candidates.len());
for (id, combined_score, semantic_score, lexical_score, recency_score, is_associative) in
scored_candidates
{
if let Some(entry) = self.entries.get_mut(&id) {
entry.last_accessed = now_ts;
entry.access_count = entry.access_count.saturating_add(1);
final_results.push(MemoryRecallResult {
entry: entry.clone(),
combined_score,
semantic_score,
lexical_score,
recency_score,
is_associative,
});
}
}
final_results
}
pub fn recall_rrf(
&mut self,
query_text: &str,
query_vector: Option<&[f32]>,
limit: usize,
rrf_k: f32,
now_ts: u64,
) -> Vec<MemoryRecallResult> {
self.recall_rrf_scoped(query_text, query_vector, limit, rrf_k, now_ts, None)
}
pub(crate) fn recall_rrf_scoped(
&mut self,
query_text: &str,
query_vector: Option<&[f32]>,
limit: usize,
rrf_k: f32,
now_ts: u64,
scope: Option<&MemoryAccessScope>,
) -> Vec<MemoryRecallResult> {
let bm25_hits = self.bm25.search(query_text, self.entries.len());
let bm25_ids: Vec<u64> = bm25_hits
.iter()
.filter_map(|(id, _)| {
let entry = self.entries.get(id)?;
if scope.is_some_and(|s| !s.matches_entry(entry))
|| entry.expires_at.is_some_and(|expiry| now_ts >= expiry)
{
return None;
}
Some(*id)
})
.collect();
let mut vector_hits: Vec<(u64, f32)> = Vec::new();
if let Some(q_vec) = query_vector {
for (id, entry) in &self.entries {
if scope.is_some_and(|s| !s.matches_entry(entry))
|| entry.expires_at.is_some_and(|expiry| now_ts >= expiry)
{
continue;
}
if let Some(e_vec) = &entry.vector
&& e_vec.len() == q_vec.len()
{
let dot: f32 = q_vec.iter().zip(e_vec.iter()).map(|(a, b)| a * b).sum();
let norm_q: f32 = q_vec.iter().map(|x| x * x).sum::<f32>().sqrt();
let norm_e: f32 = e_vec.iter().map(|x| x * x).sum::<f32>().sqrt();
let sim = if norm_q > 1e-9 && norm_e > 1e-9 {
dot / (norm_q * norm_e)
} else {
0.0
};
vector_hits.push((*id, sim));
}
}
vector_hits.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
}
let vector_ids: Vec<u64> = vector_hits.iter().map(|(id, _)| *id).collect();
let mut ranked_lists: Vec<(&[u64], f32)> = Vec::new();
if !bm25_ids.is_empty() {
ranked_lists.push((&bm25_ids, 1.0));
}
if !vector_ids.is_empty() {
ranked_lists.push((&vector_ids, 1.0));
}
let fused = bm25::reciprocal_rank_fusion(&ranked_lists, rrf_k, limit);
let mut results = Vec::new();
for (id, fused_score) in fused {
if let Some(entry) = self.entries.get_mut(&id) {
entry.last_accessed = now_ts;
entry.access_count += 1;
results.push(MemoryRecallResult {
entry: entry.clone(),
combined_score: fused_score,
semantic_score: 0.0,
lexical_score: 0.0,
recency_score: 0.0,
is_associative: false,
});
}
}
results
}
pub fn prune_decayed_ids_at(
&mut self,
decay_threshold: f32,
half_life_seconds: u64,
now_ts: u64,
) -> Vec<u64> {
self.prune_decayed_ids_at_scoped(decay_threshold, half_life_seconds, now_ts, None)
}
pub(crate) fn prune_decayed_ids_at_scoped(
&mut self,
decay_threshold: f32,
half_life_seconds: u64,
now_ts: u64,
scope: Option<&MemoryAccessScope>,
) -> Vec<u64> {
let lambda = if half_life_seconds > 0 {
2.0f32.ln() / (half_life_seconds as f32)
} else {
0.0
};
let mut to_remove: Vec<u64> = Vec::new();
for (&id, entry) in &self.entries {
if scope.is_some_and(|s| !s.matches_entry(entry)) {
continue;
}
let dt = now_ts.saturating_sub(entry.timestamp) as f32;
let recency = (-lambda * dt).exp();
let retention_score = 0.6 * entry.importance + 0.4 * recency;
if retention_score < decay_threshold {
to_remove.push(id);
}
}
for &id in &to_remove {
self.entries.remove(&id);
self.bm25.remove_document(id);
self.runtime.changed(id, None);
}
for entry in self.entries.values_mut() {
entry.associations.retain(|id| !to_remove.contains(id));
}
to_remove
}
pub fn prune_decayed_at(
&mut self,
decay_threshold: f32,
half_life_seconds: u64,
now_ts: u64,
) -> usize {
self.prune_decayed_ids_at(decay_threshold, half_life_seconds, now_ts)
.len()
}
pub fn prune(&mut self, decay_threshold: f32, half_life_seconds: u64, now_ts: u64) -> usize {
self.prune_decayed_at(decay_threshold, half_life_seconds, now_ts)
}
pub fn forget(&mut self, id: u64) -> bool {
if self.entries.remove(&id).is_some() {
self.bm25.remove_document(id);
self.runtime.changed(id, None);
for entry in self.entries.values_mut() {
entry.associations.retain(|&assoc_id| assoc_id != id);
}
true
} else {
false
}
}
pub fn get(&self, id: u64) -> Option<&MemoryEntry> {
self.entries.get(&id)
}
pub fn all_entries(&self) -> &HashMap<u64, MemoryEntry> {
&self.entries
}
}
pub fn cosine_similarity(u: &[f32], v: &[f32]) -> f32 {
if u.len() != v.len() || u.is_empty() {
return 0.0;
}
let mut dot = 0.0f64;
let mut norm_u = 0.0f64;
let mut norm_v = 0.0f64;
for i in 0..u.len() {
dot += f64::from(u[i]) * f64::from(v[i]);
norm_u += f64::from(u[i]).powi(2);
norm_v += f64::from(v[i]).powi(2);
}
let denom = norm_u.sqrt() * norm_v.sqrt();
if denom > 1e-8 {
(dot / denom).clamp(-1.0, 1.0) as f32
} else {
0.0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_memory_remember_and_hybrid_recall() {
let mut engine = MemoryEngine::new();
let now = 1_000_000;
let vec1 = vec![1.0, 0.0, 0.0];
let vec2 = vec![0.0, 1.0, 0.0];
let id1 = engine.remember(
"User prefers replies in Bahasa Melayu and lives in Kuala Lumpur",
Some(&vec1),
0.9,
&["preference", "language"],
now,
);
let id2 = engine.remember(
"User is building a high performance database engine called TapirusDB",
Some(&vec2),
0.8,
&["project", "systems"],
now,
);
assert_eq!(id1, 1);
assert_eq!(id2, 2);
let filter = MemoryRecallFilter::default();
let results = engine.recall(Some("Bahasa Melayu"), None, 5, &filter, now);
assert!(!results.is_empty());
assert_eq!(results[0].entry.id, id1);
let results_vec = engine.recall(None, Some(&vec2), 5, &filter, now);
assert!(!results_vec.is_empty());
assert_eq!(results_vec[0].entry.id, id2);
}
#[test]
fn test_temporal_recency_decay() {
let mut engine = MemoryEngine::new();
let t0 = 1_000_000;
let half_life = 86400;
let _id_old = engine.remember(
"Saw an interesting article on databases",
None,
0.5,
&[],
t0,
);
let t_now = t0 + 7 * 86400;
let id_new = engine.remember(
"Saw an interesting article on databases",
None,
0.5,
&[],
t_now,
);
let filter = MemoryRecallFilter::default()
.with_weights(0.0, 0.4, 0.6, 0.0) .with_half_life(half_life);
let results = engine.recall(Some("article databases"), None, 2, &filter, t_now);
assert_eq!(results.len(), 2);
assert_eq!(results[0].entry.id, id_new);
assert!(results[0].combined_score > results[1].combined_score);
}
#[test]
fn test_graph_associative_recall() {
let mut engine = MemoryEngine::new();
let now = 1_000_000;
let id1 = engine.remember(
"Alex Chen is the architect of TapirusDB",
None,
0.9,
&[],
now,
);
let id2 = engine.remember(
"TapirusDB uses ChaCha20-Poly1305 AEAD page encryption",
None,
0.8,
&[],
now,
);
engine.associate(id1, id2).expect("Associate memories");
let filter = MemoryRecallFilter::default().with_graph_hops(1);
let results = engine.recall(Some("Alex Chen"), None, 5, &filter, now);
assert!(results.len() >= 2);
assert_eq!(results[0].entry.id, id1);
assert_eq!(results[1].entry.id, id2);
assert!(results[1].is_associative);
}
}