mod soft_delete;
pub use soft_delete::{clear_deleted_at, find_by_name_any_state, list_deleted_before, soft_delete};
use crate::embedder::f32_to_bytes;
use crate::errors::AppError;
use crate::storage::utils::with_busy_retry;
use rusqlite::{params, Connection};
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize)]
pub struct NewMemory {
pub namespace: String,
pub name: String,
pub memory_type: String,
pub description: String,
pub body: String,
pub body_hash: String,
pub session_id: Option<String>,
pub source: String,
pub metadata: serde_json::Value,
}
#[derive(Debug, Serialize)]
pub struct MemoryRow {
pub id: i64,
pub namespace: String,
pub name: String,
pub memory_type: String,
pub description: String,
pub body: String,
pub body_hash: String,
pub session_id: Option<String>,
pub source: String,
pub metadata: String,
pub created_at: i64,
pub updated_at: i64,
#[serde(skip_serializing_if = "Option::is_none")]
pub deleted_at: Option<i64>,
}
pub fn find_by_name(
conn: &Connection,
namespace: &str,
name: &str,
) -> Result<Option<(i64, i64, i64)>, AppError> {
let mut stmt = conn.prepare_cached(
"SELECT m.id, m.updated_at, COALESCE(MAX(v.version), 0)
FROM memories m
LEFT JOIN memory_versions v ON v.memory_id = m.id
WHERE m.namespace = ?1 AND m.name = ?2 AND m.deleted_at IS NULL
GROUP BY m.id",
)?;
let result = stmt.query_row(params![namespace, name], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, i64>(1)?,
r.get::<_, i64>(2)?,
))
});
match result {
Ok(row) => Ok(Some(row)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(AppError::Database(e)),
}
}
pub fn find_by_hash(
conn: &Connection,
namespace: &str,
body_hash: &str,
) -> Result<Option<i64>, AppError> {
let mut stmt = conn.prepare_cached(
"SELECT id FROM memories WHERE namespace = ?1 AND body_hash = ?2 AND deleted_at IS NULL",
)?;
match stmt.query_row(params![namespace, body_hash], |r| r.get(0)) {
Ok(id) => Ok(Some(id)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(AppError::Database(e)),
}
}
pub fn insert(conn: &Connection, m: &NewMemory) -> Result<i64, AppError> {
let validated_source = crate::memory_source::validate_source(&m.source)?;
conn.execute(
"INSERT INTO memories (namespace, name, type, description, body, body_hash, session_id, source, metadata)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![
m.namespace, m.name, m.memory_type, m.description, m.body,
m.body_hash, m.session_id, validated_source,
serde_json::to_string(&m.metadata)?
],
)?;
Ok(conn.last_insert_rowid())
}
pub fn update(
conn: &Connection,
id: i64,
m: &NewMemory,
expected_updated_at: Option<i64>,
) -> Result<bool, AppError> {
let validated_source = crate::memory_source::validate_source(&m.source)?;
let affected = if let Some(ts) = expected_updated_at {
conn.execute(
"UPDATE memories SET type=?2, description=?3, body=?4, body_hash=?5,
session_id=?6, source=?7, metadata=?8
WHERE id=?1 AND updated_at=?9 AND deleted_at IS NULL",
params![
id,
m.memory_type,
m.description,
m.body,
m.body_hash,
m.session_id,
validated_source,
serde_json::to_string(&m.metadata)?,
ts
],
)?
} else {
conn.execute(
"UPDATE memories SET type=?2, description=?3, body=?4, body_hash=?5,
session_id=?6, source=?7, metadata=?8
WHERE id=?1 AND deleted_at IS NULL",
params![
id,
m.memory_type,
m.description,
m.body,
m.body_hash,
m.session_id,
validated_source,
serde_json::to_string(&m.metadata)?
],
)?
};
Ok(affected == 1)
}
pub fn upsert_vec(
conn: &Connection,
memory_id: i64,
namespace: &str,
_memory_type: &str,
embedding: &[f32],
_name: &str,
_snippet: &str,
) -> Result<(), AppError> {
if embedding.is_empty() {
tracing::debug!(
memory_id,
"empty memory embedding: skipping memory_embeddings row (backfill via enrich re-embed)"
);
return Ok(());
}
let embedding_bytes = f32_to_bytes(embedding);
with_busy_retry(|| {
conn.execute(
"DELETE FROM memory_embeddings WHERE memory_id = ?1",
params![memory_id],
)?;
conn.execute(
"INSERT INTO memory_embeddings(memory_id, namespace, embedding, source, model, dim)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![
memory_id,
namespace,
&embedding_bytes,
"llm-headless",
crate::constants::SQLITE_GRAPHRAG_VERSION,
crate::constants::embedding_dim() as i64,
],
)?;
Ok(())
})
}
pub fn delete_vec(conn: &Connection, memory_id: i64) -> Result<(), AppError> {
conn.execute(
"DELETE FROM memory_embeddings WHERE memory_id = ?1",
params![memory_id],
)?;
Ok(())
}
pub fn read_by_name(
conn: &Connection,
namespace: &str,
name: &str,
) -> Result<Option<MemoryRow>, AppError> {
let mut stmt = conn.prepare_cached(
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
)?;
match stmt.query_row(params![namespace, name], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
}) {
Ok(m) => Ok(Some(m)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(AppError::Database(e)),
}
}
pub fn list(
conn: &Connection,
namespace: &str,
memory_type: Option<&str>,
limit: usize,
offset: usize,
include_deleted: bool,
) -> Result<Vec<MemoryRow>, AppError> {
if let Some(mt) = memory_type {
let sql = if include_deleted {
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE namespace=?1 AND type=?2
ORDER BY updated_at DESC LIMIT ?3 OFFSET ?4"
} else {
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE namespace=?1 AND type=?2 AND deleted_at IS NULL
ORDER BY updated_at DESC LIMIT ?3 OFFSET ?4"
};
let mut stmt = conn.prepare_cached(sql)?;
let rows = stmt
.query_map(params![namespace, mt, limit as i64, offset as i64], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(rows)
} else {
let sql = if include_deleted {
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE namespace=?1
ORDER BY updated_at DESC LIMIT ?2 OFFSET ?3"
} else {
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE namespace=?1 AND deleted_at IS NULL
ORDER BY updated_at DESC LIMIT ?2 OFFSET ?3"
};
let mut stmt = conn.prepare_cached(sql)?;
let rows = stmt
.query_map(params![namespace, limit as i64, offset as i64], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(rows)
}
}
pub fn count(
conn: &Connection,
namespace: &str,
memory_type: Option<&str>,
include_deleted: bool,
) -> Result<usize, AppError> {
let (sql, params_vec): (&str, Vec<Box<dyn rusqlite::types::ToSql>>) = match (
memory_type,
include_deleted,
) {
(Some(mt), true) => (
"SELECT COUNT(*) FROM memories WHERE namespace=?1 AND type=?2",
vec![
Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>,
Box::new(mt.to_string()),
],
),
(Some(mt), false) => (
"SELECT COUNT(*) FROM memories WHERE namespace=?1 AND type=?2 AND deleted_at IS NULL",
vec![
Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>,
Box::new(mt.to_string()),
],
),
(None, true) => (
"SELECT COUNT(*) FROM memories WHERE namespace=?1",
vec![Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>],
),
(None, false) => (
"SELECT COUNT(*) FROM memories WHERE namespace=?1 AND deleted_at IS NULL",
vec![Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>],
),
};
let params_refs: Vec<&dyn rusqlite::types::ToSql> =
params_vec.iter().map(|b| b.as_ref()).collect();
let n: i64 = conn.query_row(sql, params_refs.as_slice(), |r| r.get(0))?;
Ok(n as usize)
}
pub fn knn_search(
conn: &Connection,
embedding: &[f32],
namespaces: &[String],
memory_type: Option<&str>,
k: usize,
) -> Result<Vec<(i64, f32)>, AppError> {
if embedding.len() != crate::constants::embedding_dim() {
return Err(AppError::Embedding(
crate::i18n::validation::embedding_knn_search_dim_mismatch(
embedding.len(),
crate::constants::embedding_dim(),
),
));
}
let placeholders = (0..namespaces.len())
.map(|_| "?")
.collect::<Vec<_>>()
.join(",");
let sql = if namespaces.is_empty() {
"SELECT memory_id, embedding, namespace FROM memory_embeddings".to_string()
} else {
format!(
"SELECT memory_id, embedding, namespace FROM memory_embeddings \
WHERE namespace IN ({placeholders})"
)
};
let mut stmt = conn.prepare(&sql)?;
let mut raw_params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
for ns in namespaces {
raw_params.push(Box::new(ns.clone()));
}
let param_refs: Vec<&dyn rusqlite::ToSql> = raw_params.iter().map(|b| b.as_ref()).collect();
let rows = stmt.query_map(param_refs.as_slice(), |r| {
let id: i64 = r.get(0)?;
let bytes: Vec<u8> = r.get(1)?;
let ns: String = r.get(2)?;
Ok((id, bytes, ns))
})?;
let type_filter = memory_type.map(|t| t.to_string());
let mut candidates: Vec<(i64, f32)> = Vec::new();
for row in rows {
let (id, bytes, ns) = row?;
let stored = crate::embedder::bytes_to_f32(&bytes);
if stored.len() != embedding.len() {
continue;
}
let sim = crate::similarity::cosine_similarity(embedding, &stored);
let dist = crate::similarity::similarity_to_distance(sim);
if let Some(mt) = &type_filter {
let actual: Option<String> = conn
.query_row(
"SELECT type FROM memories WHERE id = ?1",
params![id],
|r| r.get(0),
)
.ok();
if actual.as_deref() != Some(mt.as_str()) {
continue;
}
}
let _ = ns; candidates.push((id, dist));
}
candidates.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
candidates.truncate(k);
Ok(candidates)
}
pub fn read_full(conn: &Connection, memory_id: i64) -> Result<Option<MemoryRow>, AppError> {
let mut stmt = conn.prepare_cached(
"SELECT id, namespace, name, type, description, body, body_hash,
session_id, source, metadata, created_at, updated_at, deleted_at
FROM memories WHERE id=?1 AND deleted_at IS NULL",
)?;
match stmt.query_row(params![memory_id], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
}) {
Ok(m) => Ok(Some(m)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(AppError::Database(e)),
}
}
fn preprocess_fts_query(raw: &str) -> String {
const SEPARATORS: &[char] = &['-', '.', '_', '/'];
const FTS5_SYNTAX: &[char] = &['"', '*', '(', ')', '^', ':'];
const FTS5_KEYWORDS: &[&str] = &["OR", "AND", "NOT", "NEAR"];
let sanitized: String = raw.chars().filter(|c| !FTS5_SYNTAX.contains(c)).collect();
let trimmed = sanitized.trim();
if trimmed.is_empty() {
return String::new();
}
let is_fts_keyword = |t: &str| FTS5_KEYWORDS.iter().any(|kw| kw.eq_ignore_ascii_case(t));
if !trimmed.chars().any(|c| SEPARATORS.contains(&c)) {
return trimmed
.split_whitespace()
.filter(|t| !is_fts_keyword(t))
.map(|t| format!("{t}*"))
.collect::<Vec<_>>()
.join(" ");
}
let tokens: Vec<&str> = trimmed
.split(|c: char| SEPARATORS.contains(&c) || c.is_whitespace())
.filter(|t| !t.is_empty() && !is_fts_keyword(t))
.collect();
if tokens.is_empty() {
return String::new();
}
let phrase = format!("\"{}\"", tokens.join(" "));
let prefix_terms: Vec<String> = tokens.iter().map(|t| format!("{t}*")).collect();
format!("{phrase} OR {}", prefix_terms.join(" OR "))
}
pub fn fts_search(
conn: &Connection,
query: &str,
namespace: &str,
memory_type: Option<&str>,
limit: usize,
) -> Result<Vec<MemoryRow>, AppError> {
let fts_query = preprocess_fts_query(query);
if let Some(mt) = memory_type {
let mut stmt = conn.prepare_cached(
"SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
FROM fts_memories fts
JOIN memories m ON m.id = fts.rowid
WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.type = ?3 AND m.deleted_at IS NULL
ORDER BY rank LIMIT ?4",
)?;
let rows = stmt
.query_map(params![fts_query, namespace, mt, limit as i64], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(rows)
} else {
let mut stmt = conn.prepare_cached(
"SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
FROM fts_memories fts
JOIN memories m ON m.id = fts.rowid
WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.deleted_at IS NULL
ORDER BY rank LIMIT ?3",
)?;
let rows = stmt
.query_map(params![fts_query, namespace, limit as i64], |r| {
Ok(MemoryRow {
id: r.get(0)?,
namespace: r.get(1)?,
name: r.get(2)?,
memory_type: r.get(3)?,
description: r.get(4)?,
body: r.get(5)?,
body_hash: r.get(6)?,
session_id: r.get(7)?,
source: r.get(8)?,
metadata: r.get(9)?,
created_at: r.get(10)?,
updated_at: r.get(11)?,
deleted_at: r.get(12)?,
})
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(rows)
}
}
#[allow(clippy::too_many_arguments)]
pub fn sync_fts_after_update(
conn: &Connection,
memory_id: i64,
old_name: &str,
old_desc: &str,
old_body: &str,
new_name: &str,
new_desc: &str,
new_body: &str,
) -> Result<(), AppError> {
conn.execute(
"INSERT INTO fts_memories(fts_memories, rowid, name, description, body)
VALUES('delete', ?1, ?2, ?3, ?4)",
params![memory_id, old_name, old_desc, old_body],
)?;
conn.execute(
"INSERT INTO fts_memories(rowid, name, description, body)
VALUES(?1, ?2, ?3, ?4)",
params![memory_id, new_name, new_desc, new_body],
)?;
Ok(())
}
#[cfg(test)]
mod tests;