use std::collections::{BTreeSet, HashMap, HashSet};
use std::sync::RwLock;
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use tracing::{debug, info, warn};
use super::outcome::{validate_outcome_event, validate_retrieval_episode};
use super::{
apply_feedback_scores, split_for_add, FeedbackConfig, FeedbackEntry, OutcomeEvent,
OutcomePrior, RetrievalEpisode, RetrievalEpisodeId, RetrievalEpisodeItem, Store, StoreStats,
};
use crate::compaction::CompactionResult;
use crate::error::{MemdError, Result};
use crate::task_memory::{
ArtifactKind, TaskArtifact, TaskArtifactWriteResult, TaskProjection, TaskRecord,
TaskSearchFilters,
};
use crate::types::{ChunkId, ChunkStatus, MemoryChunk, TenantId};
pub struct MemoryStore {
chunks: RwLock<HashMap<String, HashMap<String, MemoryChunk>>>,
task_artifacts: RwLock<HashMap<String, HashMap<String, TaskArtifact>>>,
task_projection_links: RwLock<HashMap<String, HashMap<String, Vec<ChunkId>>>>,
projection_to_artifact: RwLock<HashMap<String, HashMap<String, String>>>,
feedback: RwLock<HashMap<String, Vec<FeedbackEntry>>>,
retrieval_episodes: RwLock<HashMap<String, (RetrievalEpisode, Vec<RetrievalEpisodeItem>)>>,
outcome_events: RwLock<HashMap<String, Vec<OutcomeEvent>>>,
}
impl MemoryStore {
pub fn new() -> Self {
Self {
chunks: RwLock::new(HashMap::new()),
task_artifacts: RwLock::new(HashMap::new()),
task_projection_links: RwLock::new(HashMap::new()),
projection_to_artifact: RwLock::new(HashMap::new()),
feedback: RwLock::new(HashMap::new()),
retrieval_episodes: RwLock::new(HashMap::new()),
outcome_events: RwLock::new(HashMap::new()),
}
}
fn compute_hash(text: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(text.as_bytes());
let result = hasher.finalize();
format!("{:x}", result)
}
async fn add_single_chunk(&self, mut chunk: MemoryChunk) -> Result<ChunkId> {
let chunk_id = ChunkId::new();
chunk.chunk_id = chunk_id.clone();
chunk.hash = Self::compute_hash(&chunk.text);
let tenant_str = chunk.tenant_id.to_string();
debug!(
tenant_id = %tenant_str,
chunk_id = %chunk_id,
chunk_type = %chunk.chunk_type,
"adding chunk to store"
);
let mut store = self.chunks.write().unwrap();
let tenant_chunks = store.entry(tenant_str).or_default();
tenant_chunks.insert(chunk_id.to_string(), chunk);
Ok(chunk_id)
}
}
impl Default for MemoryStore {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Store for MemoryStore {
async fn add(&self, chunk: MemoryChunk) -> Result<ChunkId> {
Ok(self.add_with_stored_ids(chunk).await?.0)
}
async fn add_with_stored_ids(&self, chunk: MemoryChunk) -> Result<(ChunkId, Vec<ChunkId>)> {
let mut chunks = split_for_add(chunk);
if chunks.len() == 1 {
let chunk_id = self
.add_single_chunk(chunks.pop().ok_or_else(|| {
crate::error::MemdError::StorageError("no chunk to add".into())
})?)
.await?;
return Ok((chunk_id.clone(), vec![chunk_id]));
}
let mut chunk_ids = Vec::with_capacity(chunks.len());
for chunk in chunks {
chunk_ids.push(self.add_single_chunk(chunk).await?);
}
let primary_id = chunk_ids
.first()
.cloned()
.ok_or_else(|| crate::error::MemdError::StorageError("no chunk id produced".into()))?;
Ok((primary_id, chunk_ids))
}
async fn add_batch(&self, chunks: Vec<MemoryChunk>) -> Result<Vec<ChunkId>> {
let mut ids = Vec::with_capacity(chunks.len());
info!(count = chunks.len(), "adding batch of chunks");
for chunk in chunks {
let id = self.add(chunk).await?;
ids.push(id);
}
Ok(ids)
}
async fn add_feedback(&self, feedback: FeedbackEntry) -> Result<()> {
let tenant = feedback.tenant_id.to_string();
let mut store = self.feedback.write().unwrap();
store.entry(tenant).or_default().push(feedback);
Ok(())
}
async fn add_task_artifact(
&self,
artifact: TaskArtifact,
projections: Vec<TaskProjection>,
) -> Result<TaskArtifactWriteResult> {
validate_task_artifact_write(&self.task_artifacts.read().unwrap(), &artifact)?;
let projection_chunk_ids = self
.add_batch(
projections
.into_iter()
.map(|projection| projection.chunk)
.collect(),
)
.await?
.into_iter()
.map(|chunk_id| chunk_id.to_string())
.collect::<Vec<_>>();
let tenant = artifact.tenant_id.to_string();
let artifact_id = artifact.artifact_id.clone();
let task_id = artifact.task_id.clone();
let projection_ids = projection_chunk_ids
.iter()
.filter_map(|id| ChunkId::parse(id).ok())
.collect::<Vec<_>>();
let mut task_store = self.task_artifacts.write().unwrap();
validate_task_artifact_write(&task_store, &artifact)?;
task_store
.entry(tenant.clone())
.or_default()
.insert(artifact_id.clone(), artifact);
drop(task_store);
let mut projection_store = self.task_projection_links.write().unwrap();
projection_store
.entry(tenant.clone())
.or_default()
.insert(artifact_id.clone(), projection_ids);
drop(projection_store);
let mut reverse = self.projection_to_artifact.write().unwrap();
let reverse_map = reverse.entry(tenant.clone()).or_default();
for projection_chunk_id in &projection_chunk_ids {
reverse_map.insert(projection_chunk_id.clone(), artifact_id.clone());
}
Ok(TaskArtifactWriteResult {
task_id,
artifact_id,
projection_chunk_ids,
})
}
async fn get_task_artifact(
&self,
tenant_id: &TenantId,
artifact_id: &str,
) -> Result<Option<TaskArtifact>> {
let task_store = self.task_artifacts.read().unwrap();
Ok(task_store
.get(tenant_id.as_str())
.and_then(|artifacts| artifacts.get(artifact_id))
.cloned())
}
async fn get_task_artifact_global(&self, artifact_id: &str) -> Result<Option<TaskArtifact>> {
let task_store = self.task_artifacts.read().unwrap();
let mut matches = task_store
.values()
.filter_map(|artifacts| artifacts.get(artifact_id));
let artifact = matches.next().cloned();
if matches.next().is_some() {
return Err(crate::error::MemdError::StorageError(format!(
"artifact_id '{artifact_id}' is duplicated across tenants"
)));
}
Ok(artifact)
}
async fn list_task_artifacts(
&self,
tenant_id: &TenantId,
task_id: &str,
) -> Result<Vec<TaskArtifact>> {
let task_store = self.task_artifacts.read().unwrap();
let mut artifacts = task_store
.get(tenant_id.as_str())
.map(|artifacts| {
artifacts
.values()
.filter(|artifact| artifact.task_id == task_id)
.cloned()
.collect::<Vec<_>>()
})
.unwrap_or_default();
artifacts.sort_by(|left, right| {
left.timestamp_created
.cmp(&right.timestamp_created)
.then_with(|| left.artifact_id.cmp(&right.artifact_id))
});
Ok(artifacts)
}
async fn list_thread_artifacts(
&self,
tenant_id: &TenantId,
thread_id: &str,
) -> Result<Vec<TaskArtifact>> {
let task_store = self.task_artifacts.read().unwrap();
let mut artifacts = task_store
.get(tenant_id.as_str())
.map(|artifacts| {
artifacts
.values()
.filter(|artifact| artifact.thread_key() == thread_id)
.cloned()
.collect::<Vec<_>>()
})
.unwrap_or_default();
artifacts.sort_by(|left, right| {
left.timestamp_created
.cmp(&right.timestamp_created)
.then_with(|| left.artifact_id.cmp(&right.artifact_id))
});
Ok(artifacts)
}
async fn list_tasks(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
limit: usize,
) -> Result<Vec<TaskRecord>> {
if limit == 0 {
return Ok(Vec::new());
}
let task_store = self.task_artifacts.read().unwrap();
let Some(artifacts) = task_store.get(tenant_id.as_str()) else {
return Ok(Vec::new());
};
let mut by_task: HashMap<String, TaskRecord> = HashMap::new();
for artifact in artifacts.values() {
if artifact.artifact_kind == ArtifactKind::Digest {
continue;
}
if let Some(project_id) = project_id {
if artifact.project_id.as_option() != Some(project_id) {
continue;
}
}
let record = by_task
.entry(artifact.task_id.clone())
.or_insert_with(|| TaskRecord {
task_id: artifact.task_id.clone(),
tenant_id: artifact.tenant_id.clone(),
project_id: artifact.project_id.clone(),
status: artifact.status.clone(),
goal: artifact.goal.clone(),
scientific_question: artifact.scientific_question.clone(),
hypothesis: artifact.hypothesis.clone(),
last_artifact_id: artifact.artifact_id.clone(),
started_at_ms: None,
finished_at_ms: None,
updated_at_ms: artifact.timestamp_created,
});
if record.project_id.as_option().is_none() && artifact.project_id.as_option().is_some()
{
record.project_id = artifact.project_id.clone();
}
if record.goal.is_none() {
record.goal = artifact.goal.clone();
}
if record.scientific_question.is_none() {
record.scientific_question = artifact.scientific_question.clone();
}
if record.hypothesis.is_none() {
record.hypothesis = artifact.hypothesis.clone();
}
if artifact.artifact_kind == ArtifactKind::TaskStart {
record.started_at_ms = Some(artifact.timestamp_created);
}
if artifact.artifact_kind == ArtifactKind::TaskFinish {
record.finished_at_ms = Some(artifact.timestamp_created);
}
if artifact.timestamp_created >= record.updated_at_ms {
record.updated_at_ms = artifact.timestamp_created;
record.last_artifact_id = artifact.artifact_id.clone();
record.status = artifact.status.clone().or_else(|| record.status.clone());
}
}
let mut tasks = by_task.into_values().collect::<Vec<_>>();
tasks.sort_by_key(|task| std::cmp::Reverse(task.updated_at_ms));
tasks.truncate(limit);
Ok(tasks)
}
async fn get_task_global(&self, task_id: &str) -> Result<Option<TaskRecord>> {
let tenants = {
let task_store = self.task_artifacts.read().unwrap();
task_store
.iter()
.filter(|(_, artifacts)| {
artifacts
.values()
.any(|artifact| artifact.task_id == task_id)
})
.map(|(tenant, _)| tenant.clone())
.collect::<Vec<_>>()
};
if tenants.len() > 1 {
return Err(crate::error::MemdError::StorageError(format!(
"task_id '{task_id}' is duplicated across tenants"
)));
}
let Some(tenant) = tenants.first() else {
return Ok(None);
};
let tenant = TenantId::new(tenant.clone())?;
Ok(self
.list_tasks(&tenant, None, usize::MAX)
.await?
.into_iter()
.find(|task| task.task_id == task_id))
}
async fn list_tenants(&self) -> Result<Vec<TenantId>> {
let mut names = BTreeSet::new();
for key in self.chunks.read().unwrap().keys() {
names.insert(key.clone());
}
for key in self.task_artifacts.read().unwrap().keys() {
names.insert(key.clone());
}
for key in self.feedback.read().unwrap().keys() {
names.insert(key.clone());
}
Ok(names
.into_iter()
.filter_map(|name| TenantId::new(name).ok())
.collect())
}
async fn search_task_projection_chunk_ids(
&self,
tenant_id: &TenantId,
filters: &TaskSearchFilters,
limit: usize,
) -> Result<Vec<ChunkId>> {
if limit == 0 {
return Ok(Vec::new());
}
let task_store = self.task_artifacts.read().unwrap();
let link_store = self.task_projection_links.read().unwrap();
let mut chunk_ids = Vec::new();
let Some(artifacts) = task_store.get(tenant_id.as_str()) else {
return Ok(Vec::new());
};
let links = link_store.get(tenant_id.as_str());
let mut matching = artifacts
.values()
.filter(|artifact| artifact_matches_filters(artifact, filters))
.cloned()
.collect::<Vec<_>>();
matching.sort_by_key(|artifact| std::cmp::Reverse(artifact.timestamp_created));
for artifact in matching {
if let Some(chunk_list) =
links.and_then(|tenant_links| tenant_links.get(&artifact.artifact_id))
{
for chunk_id in chunk_list {
chunk_ids.push(chunk_id.clone());
if chunk_ids.len() >= limit {
return Ok(chunk_ids);
}
}
}
}
Ok(chunk_ids)
}
async fn resolve_artifacts_for_chunks(
&self,
tenant_id: &TenantId,
chunk_ids: &[ChunkId],
) -> Result<HashMap<String, TaskArtifact>> {
let reverse = self.projection_to_artifact.read().unwrap();
let task_store = self.task_artifacts.read().unwrap();
let mut resolved = HashMap::new();
let Some(tenant_reverse) = reverse.get(tenant_id.as_str()) else {
return Ok(resolved);
};
let Some(tenant_artifacts) = task_store.get(tenant_id.as_str()) else {
return Ok(resolved);
};
for chunk_id in chunk_ids {
if let Some(artifact_id) = tenant_reverse.get(&chunk_id.to_string()) {
if let Some(artifact) = tenant_artifacts.get(artifact_id) {
resolved.insert(chunk_id.to_string(), artifact.clone());
}
}
}
Ok(resolved)
}
async fn list_feedback(
&self,
tenant_id: &TenantId,
query: &str,
limit: usize,
) -> Result<Vec<FeedbackEntry>> {
let tenant = tenant_id.to_string();
let query_hash = super::stable_query_hash(&super::normalize_query(query));
let store = self.feedback.read().unwrap();
let entries = store.get(&tenant).cloned().unwrap_or_default();
let mut filtered: Vec<FeedbackEntry> = entries
.into_iter()
.filter(|entry| entry.query_hash == query_hash)
.collect();
filtered.sort_by_key(|entry| std::cmp::Reverse(entry.timestamp_ms));
filtered.truncate(limit);
Ok(filtered)
}
async fn record_retrieval_episode(
&self,
episode: RetrievalEpisode,
items: Vec<RetrievalEpisodeItem>,
) -> Result<()> {
validate_retrieval_episode(&episode, &items)?;
let chunks = self.chunks.read().unwrap();
for item in &items {
let tenant_chunks = chunks.get(item.origin_tenant_id.as_str()).ok_or_else(|| {
crate::error::MemdError::ValidationError(
"retrieval episode item origin tenant has no chunks".to_string(),
)
})?;
let chunk = tenant_chunks
.get(&item.chunk_id.to_string())
.ok_or_else(|| {
crate::error::MemdError::ValidationError(format!(
"retrieval episode chunk {} does not belong to its origin tenant",
item.chunk_id
))
})?;
if chunk.project_id.as_option() != item.origin_project_id.as_deref() {
return Err(crate::error::MemdError::ValidationError(format!(
"retrieval episode chunk {} origin project does not match",
item.chunk_id
)));
}
}
drop(chunks);
let key = episode.episode_id.to_string();
let mut episodes = self.retrieval_episodes.write().unwrap();
if episodes.contains_key(&key) {
return Err(crate::error::MemdError::ValidationError(format!(
"retrieval episode {} already exists",
episode.episode_id
)));
}
episodes.insert(key, (episode, items));
Ok(())
}
async fn get_retrieval_episode(
&self,
tenant_id: &TenantId,
episode_id: &RetrievalEpisodeId,
) -> Result<Option<(RetrievalEpisode, Vec<RetrievalEpisodeItem>)>> {
Ok(self
.retrieval_episodes
.read()
.unwrap()
.get(&episode_id.to_string())
.filter(|(episode, _)| &episode.tenant_id == tenant_id)
.cloned())
}
async fn finalize_retrieval_episode(
&self,
tenant_id: &TenantId,
episode_id: &RetrievalEpisodeId,
rendered_chunk_ids: &[ChunkId],
) -> Result<()> {
let mut episodes = self.retrieval_episodes.write().unwrap();
let (episode, items) = episodes
.get_mut(&episode_id.to_string())
.filter(|(episode, _)| &episode.tenant_id == tenant_id)
.ok_or_else(|| {
crate::error::MemdError::ValidationError(format!(
"unknown retrieval episode {episode_id} for tenant"
))
})?;
super::apply_rendered_order(episode, items, rendered_chunk_ids)
}
async fn record_outcome(&self, tenant_id: &TenantId, event: OutcomeEvent) -> Result<()> {
let (episode, items) = self
.get_retrieval_episode(tenant_id, &event.episode_id)
.await?
.ok_or_else(|| {
crate::error::MemdError::ValidationError(format!(
"unknown retrieval episode {} for tenant",
event.episode_id
))
})?;
validate_outcome_event(tenant_id, &episode, &items, &event)?;
let mut outcomes = self.outcome_events.write().unwrap();
let events = outcomes.entry(event.episode_id.to_string()).or_default();
if events
.iter()
.any(|existing| existing.event_id == event.event_id)
{
return Err(crate::error::MemdError::ValidationError(format!(
"outcome event {} already exists",
event.event_id
)));
}
events.push(event);
Ok(())
}
async fn list_outcomes_for_episode(
&self,
tenant_id: &TenantId,
episode_id: &RetrievalEpisodeId,
) -> Result<Vec<OutcomeEvent>> {
if self
.get_retrieval_episode(tenant_id, episode_id)
.await?
.is_none()
{
return Ok(Vec::new());
}
Ok(self
.outcome_events
.read()
.unwrap()
.get(&episode_id.to_string())
.cloned()
.unwrap_or_default())
}
async fn outcome_priors(
&self,
scope_tenant_id: &TenantId,
scope_project_id: Option<&str>,
chunk_ids: &[ChunkId],
now_ms: i64,
) -> Result<Vec<OutcomePrior>> {
let requested = chunk_ids.iter().cloned().collect::<HashSet<_>>();
if requested.is_empty() {
return Ok(Vec::new());
}
let episodes = self.retrieval_episodes.read().unwrap();
let outcomes = self.outcome_events.read().unwrap();
let mut priors = HashMap::<ChunkId, OutcomePrior>::new();
let mut seen_episode_chunks = HashSet::<(String, ChunkId)>::new();
for (episode_key, (episode, items)) in episodes.iter() {
if &episode.tenant_id != scope_tenant_id
|| episode.project_id.as_deref() != scope_project_id
{
continue;
}
let eligible_items = items
.iter()
.filter(|item| requested.contains(&item.chunk_id))
.map(|item| item.chunk_id.clone())
.collect::<HashSet<_>>();
if eligible_items.is_empty() {
continue;
}
let mut events = outcomes.get(episode_key).cloned().unwrap_or_default();
events.sort_by_key(|event| std::cmp::Reverse(event.timestamp_ms));
for event in events
.into_iter()
.filter(|event| event.contributes_to_ranking() && event.timestamp_ms <= now_ms)
{
let (positive, attributed) = if event.outcome.credits_used() {
(true, &event.used_chunk_ids)
} else if event.outcome.credits_harmful() {
(false, &event.harmful_chunk_ids)
} else {
continue;
};
for chunk_id in attributed {
if !eligible_items.contains(chunk_id)
|| !seen_episode_chunks.insert((episode_key.clone(), chunk_id.clone()))
{
continue;
}
priors
.entry(chunk_id.clone())
.or_insert_with(|| OutcomePrior::new(chunk_id.clone()))
.add(
positive,
super::decayed_outcome_weight(event.timestamp_ms, now_ms),
event.timestamp_ms,
);
}
}
}
let mut priors = priors.into_values().collect::<Vec<_>>();
priors.sort_by_key(|prior| prior.chunk_id.to_string());
Ok(priors)
}
async fn get(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<Option<MemoryChunk>> {
let tenant_str = tenant_id.to_string();
let chunk_id_str = chunk_id.to_string();
debug!(
tenant_id = %tenant_str,
chunk_id = %chunk_id_str,
"getting chunk from store"
);
let store = self.chunks.read().unwrap();
let chunk = store
.get(&tenant_str)
.and_then(|tenant_chunks| tenant_chunks.get(&chunk_id_str))
.filter(|c| {
!matches!(
c.status,
ChunkStatus::Candidate | ChunkStatus::Deleted | ChunkStatus::Error
)
})
.cloned();
Ok(chunk)
}
async fn search(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
) -> Result<Vec<MemoryChunk>> {
let tenant_str = tenant_id.to_string();
debug!(
tenant_id = %tenant_str,
query = %query,
k = k,
"searching chunks"
);
let store = self.chunks.read().unwrap();
let results: Vec<MemoryChunk> = store
.get(&tenant_str)
.map(|tenant_chunks| {
tenant_chunks
.values()
.filter(|chunk| {
!matches!(
chunk.status,
ChunkStatus::Candidate | ChunkStatus::Deleted | ChunkStatus::Error
)
})
.filter(|chunk| {
query.is_empty()
|| chunk.text.to_lowercase().contains(&query.to_lowercase())
})
.take(k)
.cloned()
.collect()
})
.unwrap_or_default();
info!(
tenant_id = %tenant_str,
query = %query,
results_count = results.len(),
"search completed"
);
Ok(results)
}
async fn search_with_scores(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
) -> Result<Vec<(MemoryChunk, f32)>> {
let chunks = self.search(tenant_id, query, k).await?;
let scored: Vec<(MemoryChunk, f32)> =
chunks.into_iter().map(|chunk| (chunk, 1.0)).collect();
let feedback = self.list_feedback(tenant_id, query, 512).await?;
Ok(apply_feedback_scores(
scored,
query,
&feedback,
current_time_ms(),
&FeedbackConfig::default(),
))
}
async fn search_with_scores_at(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
ranking_time_ms: i64,
) -> Result<Vec<(MemoryChunk, f32)>> {
let chunks = self.search(tenant_id, query, k).await?;
let scored = chunks.into_iter().map(|chunk| (chunk, 1.0)).collect();
let feedback = self.list_feedback(tenant_id, query, 512).await?;
Ok(apply_feedback_scores(
scored,
query,
&feedback,
ranking_time_ms,
&FeedbackConfig::default(),
))
}
async fn delete(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<bool> {
let tenant_str = tenant_id.to_string();
let chunk_id_str = chunk_id.to_string();
debug!(
tenant_id = %tenant_str,
chunk_id = %chunk_id_str,
"deleting chunk (soft delete)"
);
let mut store = self.chunks.write().unwrap();
if let Some(tenant_chunks) = store.get_mut(&tenant_str) {
if let Some(chunk) = tenant_chunks.get_mut(&chunk_id_str) {
if chunk.status == ChunkStatus::Deleted {
warn!(
tenant_id = %tenant_str,
chunk_id = %chunk_id_str,
"chunk already deleted"
);
return Ok(false);
}
chunk.status = ChunkStatus::Deleted;
info!(
tenant_id = %tenant_str,
chunk_id = %chunk_id_str,
"chunk marked as deleted"
);
return Ok(true);
}
}
warn!(
tenant_id = %tenant_str,
chunk_id = %chunk_id_str,
"chunk not found for deletion"
);
Ok(false)
}
async fn stats(&self, tenant_id: &TenantId) -> Result<StoreStats> {
let tenant_str = tenant_id.to_string();
debug!(tenant_id = %tenant_str, "getting store stats");
let store = self.chunks.read().unwrap();
let stats = store
.get(&tenant_str)
.map(|tenant_chunks| {
let mut chunk_types_active: HashMap<String, usize> = HashMap::new();
let mut chunk_types_deleted: HashMap<String, usize> = HashMap::new();
let mut chunk_types_all: HashMap<String, usize> = HashMap::new();
let mut deleted_count = 0;
let mut active_count = 0;
let mut candidate_count = 0;
for chunk in tenant_chunks.values() {
let key = chunk.chunk_type.to_string();
*chunk_types_all.entry(key.clone()).or_insert(0) += 1;
if chunk.status == ChunkStatus::Deleted {
deleted_count += 1;
*chunk_types_deleted.entry(key).or_insert(0) += 1;
} else if chunk.status == ChunkStatus::Candidate {
candidate_count += 1;
} else if chunk.status != ChunkStatus::Error {
active_count += 1;
*chunk_types_active.entry(key).or_insert(0) += 1;
}
}
StoreStats {
total_chunks: tenant_chunks.len(),
active_chunks: active_count,
candidate_chunks: candidate_count,
deleted_chunks: deleted_count,
chunk_types: chunk_types_active.clone(),
chunk_types_active,
chunk_types_deleted,
chunk_types_all,
}
})
.unwrap_or_default();
Ok(stats)
}
async fn list_chunks(
&self,
tenant_id: &TenantId,
limit: usize,
offset: usize,
) -> Result<Vec<MemoryChunk>> {
if limit == 0 {
return Ok(Vec::new());
}
let tenant_str = tenant_id.to_string();
let store = self.chunks.read().unwrap();
let mut chunks: Vec<MemoryChunk> = store
.get(&tenant_str)
.map(|tenant_chunks| {
tenant_chunks
.values()
.filter(|chunk| {
!matches!(
chunk.status,
ChunkStatus::Candidate | ChunkStatus::Deleted | ChunkStatus::Error
)
})
.cloned()
.collect()
})
.unwrap_or_default();
chunks.sort_by_key(|chunk| std::cmp::Reverse(chunk.timestamp_created));
if offset >= chunks.len() {
return Ok(Vec::new());
}
Ok(chunks.into_iter().skip(offset).take(limit).collect())
}
fn run_compaction_if_needed(&self, _tenant_id: &TenantId) -> Result<Option<CompactionResult>> {
Ok(None)
}
}
fn validate_task_artifact_write(
task_store: &HashMap<String, HashMap<String, TaskArtifact>>,
incoming: &TaskArtifact,
) -> Result<()> {
for artifacts in task_store.values() {
if let Some(existing) = artifacts.get(&incoming.artifact_id) {
if existing != incoming
&& (existing.experience.is_some() || incoming.experience.is_some())
{
return Err(MemdError::ValidationError(format!(
"artifact_id collision for '{}': experience artifacts cannot be overwritten",
incoming.artifact_id
)));
}
}
if incoming.artifact_kind != ArtifactKind::Digest
&& artifacts.values().any(|artifact| {
artifact.artifact_kind != ArtifactKind::Digest
&& artifact.task_id == incoming.task_id
&& artifact.tenant_id != incoming.tenant_id
})
{
return Err(MemdError::ValidationError(format!(
"task_id collision for '{}': existing tenant differs from '{}'",
incoming.task_id, incoming.tenant_id
)));
}
}
Ok(())
}
fn current_time_ms() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
}
fn artifact_matches_filters(artifact: &TaskArtifact, filters: &TaskSearchFilters) -> bool {
if let Some(task_id) = filters.task_id.as_deref() {
if artifact.task_id != task_id {
return false;
}
}
if let Some(kind) = filters.artifact_kind {
if artifact.artifact_kind != kind {
return false;
}
}
if let Some(status) = filters.status.as_deref() {
if artifact.status.as_deref() != Some(status) {
return false;
}
}
if let Some(challenge_id) = filters.challenge_id.as_deref() {
if artifact.challenge_id.as_deref() != Some(challenge_id) {
return false;
}
}
if let Some(thread_id) = filters.thread_id.as_deref() {
if artifact.thread_key() != thread_id {
return false;
}
}
if let Some(reply_to_artifact_id) = filters.reply_to_artifact_id.as_deref() {
if artifact.reply_to_artifact_id.as_deref() != Some(reply_to_artifact_id) {
return false;
}
}
if let Some(artifact_role) = filters.artifact_role.as_deref() {
if artifact.artifact_role.as_deref() != Some(artifact_role) {
return false;
}
}
if let Some(project_id) = filters.project_id.as_deref() {
if artifact.project_id.as_option() != Some(project_id) {
return false;
}
}
if let Some(agent_id) = filters.agent_id.as_deref() {
if artifact.agent_id.as_deref() != Some(agent_id) {
return false;
}
}
if let Some(session_id) = filters.session_id.as_deref() {
if artifact.session_id.as_deref() != Some(session_id) {
return false;
}
}
if let Some(tool_name) = filters.tool_name.as_deref() {
if artifact.tool_name.as_deref() != Some(tool_name)
&& artifact.provenance.tool_name.as_deref() != Some(tool_name)
{
return false;
}
}
if let Some(requested_action) = filters.requested_action.as_deref() {
if artifact.requested_action.as_deref() != Some(requested_action) {
return false;
}
}
if let Some(verification_status) = filters.verification_status.as_deref() {
if artifact.verification_status.as_deref() != Some(verification_status) {
return false;
}
}
if let Some(dataset_name) = filters.dataset_name.as_deref() {
let has_match = artifact.dataset_refs.iter().any(|dataset| {
dataset.name == dataset_name
&& filters
.dataset_version
.as_deref()
.map(|version| dataset.version.as_deref() == Some(version))
.unwrap_or(true)
});
if !has_match {
return false;
}
} else if let Some(dataset_version) = filters.dataset_version.as_deref() {
if !artifact
.dataset_refs
.iter()
.any(|dataset| dataset.version.as_deref() == Some(dataset_version))
{
return false;
}
}
if let Some(entity_name) = filters.entity_name.as_deref() {
let has_match = artifact.entity_refs.iter().any(|entity| {
entity.name == entity_name
&& filters
.entity_type
.as_deref()
.map(|entity_type| entity.entity_type == entity_type)
.unwrap_or(true)
});
if !has_match {
return false;
}
} else if let Some(entity_type) = filters.entity_type.as_deref() {
if !artifact
.entity_refs
.iter()
.any(|entity| entity.entity_type == entity_type)
{
return false;
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ChunkType;
fn make_tenant() -> TenantId {
TenantId::new("test_tenant").unwrap()
}
fn make_chunk(tenant: &TenantId, text: &str, chunk_type: ChunkType) -> MemoryChunk {
MemoryChunk::new(tenant.clone(), text, chunk_type)
}
fn make_long_document() -> String {
let sentence =
"This is a long test sentence that should trigger document chunking behavior. ";
sentence.repeat(40)
}
#[tokio::test]
async fn add_and_get_chunk() {
let store = MemoryStore::new();
let tenant = make_tenant();
let chunk = make_chunk(&tenant, "hello world", ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let retrieved = store.get(&tenant, &chunk_id).await.unwrap();
assert!(retrieved.is_some());
let retrieved = retrieved.unwrap();
assert_eq!(retrieved.text, "hello world");
assert_eq!(retrieved.chunk_type, ChunkType::Doc);
}
#[tokio::test]
async fn chunk_id_is_uuidv7() {
let store = MemoryStore::new();
let tenant = make_tenant();
let chunk = make_chunk(&tenant, "test", ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let uuid_str = chunk_id.to_string();
assert!(uuid::Uuid::parse_str(&uuid_str).is_ok());
}
#[tokio::test]
async fn content_hash_is_sha256() {
let store = MemoryStore::new();
let tenant = make_tenant();
let text = "test content";
let chunk = make_chunk(&tenant, text, ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let retrieved = store.get(&tenant, &chunk_id).await.unwrap().unwrap();
assert_eq!(retrieved.hash.len(), 64);
assert!(retrieved.hash.chars().all(|c| c.is_ascii_hexdigit()));
let expected = MemoryStore::compute_hash(text);
assert_eq!(retrieved.hash, expected);
}
#[tokio::test]
async fn tenant_isolation() {
let store = MemoryStore::new();
let tenant_a = TenantId::new("tenant_a").unwrap();
let tenant_b = TenantId::new("tenant_b").unwrap();
let chunk = make_chunk(&tenant_a, "secret data", ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let from_a = store.get(&tenant_a, &chunk_id).await.unwrap();
assert!(from_a.is_some());
let from_b = store.get(&tenant_b, &chunk_id).await.unwrap();
assert!(from_b.is_none());
}
#[tokio::test]
async fn search_returns_matching_chunks() {
let store = MemoryStore::new();
let tenant = make_tenant();
store
.add(make_chunk(&tenant, "hello world", ChunkType::Doc))
.await
.unwrap();
store
.add(make_chunk(&tenant, "goodbye world", ChunkType::Doc))
.await
.unwrap();
store
.add(make_chunk(&tenant, "other content", ChunkType::Code))
.await
.unwrap();
let results = store.search(&tenant, "world", 10).await.unwrap();
assert_eq!(results.len(), 2);
}
#[tokio::test]
async fn search_respects_k_limit() {
let store = MemoryStore::new();
let tenant = make_tenant();
for i in 0..10 {
store
.add(make_chunk(&tenant, &format!("chunk {}", i), ChunkType::Doc))
.await
.unwrap();
}
let results = store.search(&tenant, "", 5).await.unwrap();
assert_eq!(results.len(), 5);
}
#[tokio::test]
async fn search_tenant_isolation() {
let store = MemoryStore::new();
let tenant_a = TenantId::new("tenant_a").unwrap();
let tenant_b = TenantId::new("tenant_b").unwrap();
store
.add(make_chunk(&tenant_a, "secret data", ChunkType::Doc))
.await
.unwrap();
let results = store.search(&tenant_b, "secret", 10).await.unwrap();
assert!(results.is_empty());
}
#[tokio::test]
async fn soft_delete() {
let store = MemoryStore::new();
let tenant = make_tenant();
let chunk = make_chunk(&tenant, "to be deleted", ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let deleted = store.delete(&tenant, &chunk_id).await.unwrap();
assert!(deleted);
let retrieved = store.get(&tenant, &chunk_id).await.unwrap();
assert!(retrieved.is_none());
let results = store.search(&tenant, "deleted", 10).await.unwrap();
assert!(results.is_empty());
}
#[tokio::test]
async fn delete_tenant_isolation() {
let store = MemoryStore::new();
let tenant_a = TenantId::new("tenant_a").unwrap();
let tenant_b = TenantId::new("tenant_b").unwrap();
let chunk = make_chunk(&tenant_a, "protected data", ChunkType::Doc);
let chunk_id = store.add(chunk).await.unwrap();
let deleted = store.delete(&tenant_b, &chunk_id).await.unwrap();
assert!(!deleted);
let retrieved = store.get(&tenant_a, &chunk_id).await.unwrap();
assert!(retrieved.is_some());
}
#[tokio::test]
async fn stats_counts_correctly() {
let store = MemoryStore::new();
let tenant = make_tenant();
store
.add(make_chunk(&tenant, "doc 1", ChunkType::Doc))
.await
.unwrap();
store
.add(make_chunk(&tenant, "doc 2", ChunkType::Doc))
.await
.unwrap();
let code_id = store
.add(make_chunk(&tenant, "code 1", ChunkType::Code))
.await
.unwrap();
store.delete(&tenant, &code_id).await.unwrap();
let stats = store.stats(&tenant).await.unwrap();
assert_eq!(stats.total_chunks, 3);
assert_eq!(stats.active_chunks, 2);
assert_eq!(stats.deleted_chunks, 1);
assert_eq!(stats.chunk_types.get("doc"), Some(&2));
assert_eq!(stats.chunk_types.get("code"), None);
assert_eq!(stats.chunk_types_active.get("doc"), Some(&2));
assert_eq!(stats.chunk_types_deleted.get("code"), Some(&1));
assert_eq!(stats.chunk_types_all.get("code"), Some(&1));
}
#[tokio::test]
async fn add_batch() {
let store = MemoryStore::new();
let tenant = make_tenant();
let chunks = vec![
make_chunk(&tenant, "batch 1", ChunkType::Doc),
make_chunk(&tenant, "batch 2", ChunkType::Code),
make_chunk(&tenant, "batch 3", ChunkType::Trace),
];
let ids = store.add_batch(chunks).await.unwrap();
assert_eq!(ids.len(), 3);
for id in ids {
let chunk = store.get(&tenant, &id).await.unwrap();
assert!(chunk.is_some());
}
}
#[tokio::test]
async fn add_long_document_splits_into_multiple_chunks() {
let store = MemoryStore::new();
let tenant = make_tenant();
let long_text = make_long_document();
let _chunk_id = store
.add(make_chunk(&tenant, &long_text, ChunkType::Doc))
.await
.unwrap();
let stats = store.stats(&tenant).await.unwrap();
assert!(stats.total_chunks > 1);
}
#[tokio::test]
async fn feedback_adjusts_search_scores() {
let store = MemoryStore::new();
let tenant = make_tenant();
let alpha = store
.add(make_chunk(&tenant, "alpha parser notes", ChunkType::Doc))
.await
.unwrap();
let beta = store
.add(make_chunk(&tenant, "beta parser notes", ChunkType::Doc))
.await
.unwrap();
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0);
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
alpha.clone(),
crate::store::RelevanceLabel::Relevant,
now_ms,
))
.await
.unwrap();
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
alpha.clone(),
crate::store::RelevanceLabel::Relevant,
now_ms,
))
.await
.unwrap();
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
beta.clone(),
crate::store::RelevanceLabel::Irrelevant,
now_ms,
))
.await
.unwrap();
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
beta.clone(),
crate::store::RelevanceLabel::Irrelevant,
now_ms,
))
.await
.unwrap();
let ranked = store
.search_with_scores(&tenant, "parser notes", 10)
.await
.unwrap();
assert_eq!(ranked.len(), 2);
assert_eq!(ranked[0].0.chunk_id, alpha);
}
#[tokio::test]
async fn fixed_ranking_time_pins_feedback_decay() {
let store = MemoryStore::new();
let tenant = make_tenant();
let chunk_id = store
.add(make_chunk(&tenant, "parser notes", ChunkType::Doc))
.await
.unwrap();
let ranking_time_ms = 1_700_000_000_000i64;
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
chunk_id.clone(),
crate::store::RelevanceLabel::Relevant,
ranking_time_ms,
))
.await
.unwrap();
store
.add_feedback(FeedbackEntry::new(
tenant.clone(),
"parser notes",
chunk_id.clone(),
crate::store::RelevanceLabel::Irrelevant,
ranking_time_ms + 30 * 24 * 60 * 60 * 1_000,
))
.await
.unwrap();
let first = store
.search_with_scores_at(&tenant, "parser notes", 10, ranking_time_ms)
.await
.unwrap();
let second = store
.search_with_scores_at(&tenant, "parser notes", 10, ranking_time_ms)
.await
.unwrap();
let later = store
.search_with_scores_at(
&tenant,
"parser notes",
10,
ranking_time_ms + 30 * 24 * 60 * 60 * 1_000,
)
.await
.unwrap();
assert_eq!(first[0].1.to_bits(), second[0].1.to_bits());
assert!(first[0].1 > later[0].1);
}
#[tokio::test]
async fn empty_tenant_stats() {
let store = MemoryStore::new();
let tenant = make_tenant();
let stats = store.stats(&tenant).await.unwrap();
assert_eq!(stats.total_chunks, 0);
assert_eq!(stats.deleted_chunks, 0);
assert!(stats.chunk_types.is_empty());
}
}