pub mod dense;
pub mod feedback;
pub mod hybrid;
pub mod memory;
pub mod metadata;
pub mod persistent;
pub mod segment;
pub mod shared_add;
pub mod supersession;
pub mod tenant;
pub mod tombstone;
pub mod usage;
pub mod wal;
pub mod writer_lock;
use std::collections::HashMap;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::compaction::{CompactionMetrics, CompactionResult};
use crate::error::{MemdError, Result};
use crate::metrics::IndexStats;
use crate::task_memory::{
TaskArtifact, TaskArtifactWriteResult, TaskProjection, TaskRecord, TaskSearchFilters,
};
use crate::tiered::TieredTiming;
use crate::types::lifecycle::{LifecycleMetadata, ResolvedChunk};
use crate::types::{ChunkId, MemoryChunk, TenantId};
pub use feedback::{
apply_feedback_scores, normalize_query, FeedbackConfig, FeedbackEntry, RelevanceLabel,
};
#[derive(Debug, Clone, Default)]
pub struct StoreStats {
pub total_chunks: usize,
pub active_chunks: usize,
pub deleted_chunks: usize,
pub chunk_types: HashMap<String, usize>,
pub chunk_types_active: HashMap<String, usize>,
pub chunk_types_deleted: HashMap<String, usize>,
pub chunk_types_all: HashMap<String, usize>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct HealthCounts {
pub active_chunks: usize,
pub deleted_chunks: usize,
pub expired_chunks: usize,
pub superseded_chunks: usize,
pub history_chunks: usize,
pub total_chunks: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DuplicateExample {
pub canonical_text_preview: String,
pub count: usize,
pub byte_count: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DuplicateHealth {
pub unique_text_count: usize,
pub exact_duplicate_group_count: usize,
pub duplicate_row_count: usize,
pub duplicate_row_ratio: f64,
pub duplicate_byte_ratio: f64,
#[serde(default)]
pub examples: Vec<DuplicateExample>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct IndexCoverageHealth {
pub pending: usize,
pub indexed: usize,
pub failed: usize,
pub indexed_percentage: f64,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PayloadHealth {
pub p50_canonical_text_bytes: usize,
pub p95_canonical_text_bytes: usize,
pub max_canonical_text_bytes: usize,
pub p95_artifact_json_bytes: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct StoreHealthSnapshot {
pub counts: HealthCounts,
pub chunk_types_active: HashMap<String, usize>,
pub chunk_types_all: HashMap<String, usize>,
pub duplicates: DuplicateHealth,
pub index_coverage: IndexCoverageHealth,
pub payload: PayloadHealth,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExternalMutationOutcome {
Unavailable,
Clean,
OwnWrites,
External { repaired: bool },
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct RywProbeStats {
pub checks: u64,
pub external_detected: u64,
pub repairs: u64,
}
pub(crate) fn score_candidate_chunk(query: &str, chunk: &MemoryChunk) -> f32 {
if query.trim().is_empty() {
return 1.0;
}
let haystack = format!("{} {}", chunk.text, chunk.tags.join(" ")).to_ascii_lowercase();
let terms = query
.split_whitespace()
.map(|term| term.to_ascii_lowercase())
.collect::<Vec<_>>();
if terms.is_empty() {
return 1.0;
}
let mut score = 0.0f32;
for term in terms {
if haystack.contains(&term) {
score += 1.0;
}
}
if haystack.contains(&query.to_ascii_lowercase()) {
score += 1.5;
}
score
}
pub(crate) fn rank_candidate_chunks(
mut chunks: Vec<MemoryChunk>,
query: &str,
k: usize,
) -> Vec<(MemoryChunk, f32)> {
let mut scored = chunks
.drain(..)
.map(|chunk| {
let score = score_candidate_chunk(query, &chunk);
(chunk, score)
})
.collect::<Vec<_>>();
scored.sort_by(|(left_chunk, left_score), (right_chunk, right_score)| {
right_score
.partial_cmp(left_score)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| {
right_chunk
.timestamp_created
.cmp(&left_chunk.timestamp_created)
})
});
if !query.trim().is_empty() {
scored.retain(|(_, score)| *score > 0.0);
}
scored.truncate(k);
scored
}
#[async_trait]
pub trait Store: Send + Sync {
async fn add(&self, chunk: MemoryChunk) -> Result<ChunkId>;
async fn add_batch(&self, chunks: Vec<MemoryChunk>) -> Result<Vec<ChunkId>>;
async fn add_task_artifact(
&self,
_artifact: TaskArtifact,
_projections: Vec<TaskProjection>,
) -> Result<TaskArtifactWriteResult> {
Err(MemdError::StorageError(
"task artifacts not supported by this store".into(),
))
}
async fn get_task_artifact(
&self,
_tenant_id: &TenantId,
_artifact_id: &str,
) -> Result<Option<TaskArtifact>> {
Ok(None)
}
async fn list_task_artifacts(
&self,
_tenant_id: &TenantId,
_task_id: &str,
) -> Result<Vec<TaskArtifact>> {
Ok(Vec::new())
}
async fn list_thread_artifacts(
&self,
_tenant_id: &TenantId,
_thread_id: &str,
) -> Result<Vec<TaskArtifact>> {
Ok(Vec::new())
}
async fn list_tasks(
&self,
_tenant_id: &TenantId,
_project_id: Option<&str>,
_limit: usize,
) -> Result<Vec<TaskRecord>> {
Ok(Vec::new())
}
async fn list_tenants(&self) -> Result<Vec<TenantId>> {
Ok(Vec::new())
}
async fn search_task_projection_chunk_ids(
&self,
_tenant_id: &TenantId,
_filters: &TaskSearchFilters,
_limit: usize,
) -> Result<Vec<ChunkId>> {
Ok(Vec::new())
}
async fn rerank_chunks_for_query(
&self,
tenant_id: &TenantId,
query: &str,
chunk_ids: &[ChunkId],
k: usize,
) -> Result<Vec<(MemoryChunk, f32)>> {
let mut chunks = Vec::with_capacity(chunk_ids.len());
for chunk_id in chunk_ids {
if let Some(chunk) = self.get(tenant_id, chunk_id).await? {
chunks.push(chunk);
}
}
Ok(rank_candidate_chunks(chunks, query, k))
}
async fn resolve_artifacts_for_chunks(
&self,
_tenant_id: &TenantId,
_chunk_ids: &[ChunkId],
) -> Result<HashMap<String, TaskArtifact>> {
Ok(HashMap::new())
}
async fn add_feedback(&self, _feedback: FeedbackEntry) -> Result<()> {
Ok(())
}
async fn list_feedback(
&self,
_tenant_id: &TenantId,
_query: &str,
_limit: usize,
) -> Result<Vec<FeedbackEntry>> {
Ok(Vec::new())
}
async fn get(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<Option<MemoryChunk>>;
async fn get_with_lifecycle(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
) -> Result<Option<ResolvedChunk>> {
Ok(self
.get(tenant_id, chunk_id)
.await?
.map(|chunk| ResolvedChunk {
status: chunk.status,
chunk,
lifecycle: LifecycleMetadata::default(),
}))
}
fn as_persistent(&self) -> Option<&crate::store::persistent::PersistentStore> {
None
}
fn record_usage_event(&self, _event: crate::store::usage::UsageEvent) {}
async fn probe_external_mutation(&self) -> ExternalMutationOutcome {
ExternalMutationOutcome::Unavailable
}
fn ryw_probe_stats(&self) -> Option<RywProbeStats> {
None
}
async fn search(&self, tenant_id: &TenantId, query: &str, k: usize)
-> Result<Vec<MemoryChunk>>;
async fn search_with_scores(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
) -> Result<Vec<(MemoryChunk, f32)>> {
let chunks = self.search(tenant_id, query, k).await?;
Ok(chunks.into_iter().map(|c| (c, 1.0)).collect())
}
fn retrieval_mode(&self) -> &'static str {
"text_fallback"
}
async fn list_chunks(
&self,
tenant_id: &TenantId,
limit: usize,
offset: usize,
) -> Result<Vec<MemoryChunk>> {
if limit == 0 {
return Ok(Vec::new());
}
let fetch = limit.saturating_add(offset);
let mut chunks = self.search(tenant_id, "", fetch).await?;
if offset >= chunks.len() {
return Ok(Vec::new());
}
Ok(chunks.drain(offset..).take(limit).collect())
}
async fn list_chunks_for_project(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
limit: usize,
offset: usize,
) -> Result<Vec<MemoryChunk>> {
if limit == 0 {
return Ok(Vec::new());
}
if project_id.is_none() {
return self.list_chunks(tenant_id, limit, offset).await;
}
let page_size = limit.clamp(1, 500);
let target = offset.saturating_add(limit);
let mut raw_offset = 0usize;
let mut filtered = Vec::new();
while filtered.len() < target {
let chunks = self.list_chunks(tenant_id, page_size, raw_offset).await?;
if chunks.is_empty() {
break;
}
raw_offset = raw_offset.saturating_add(page_size);
filtered.extend(
chunks
.into_iter()
.filter(|chunk| chunk.project_id.as_option() == project_id),
);
}
if offset >= filtered.len() {
return Ok(Vec::new());
}
Ok(filtered.into_iter().skip(offset).take(limit).collect())
}
async fn delete(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<bool>;
async fn stats(&self, tenant_id: &TenantId) -> Result<StoreStats>;
async fn health_snapshot(
&self,
_tenant_id: &TenantId,
_project_id: Option<&str>,
_duplicate_limit: usize,
) -> Result<Option<StoreHealthSnapshot>> {
Ok(None)
}
async fn search_with_tier_info(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
) -> Result<(Vec<(MemoryChunk, f32)>, Option<TieredTiming>)> {
let results = self.search_with_scores(tenant_id, query, k).await?;
Ok((results, None))
}
fn get_tiered_stats(&self) -> Option<persistent::TieredStats> {
None
}
fn get_index_stats(&self, _tenant_id: Option<&TenantId>) -> HashMap<String, IndexStats> {
HashMap::new()
}
fn run_compaction(&self, _tenant_id: &TenantId) -> Result<CompactionResult> {
Err(MemdError::StorageError("compaction not supported".into()))
}
fn run_compaction_if_needed(&self, _tenant_id: &TenantId) -> Result<Option<CompactionResult>> {
Ok(None)
}
fn get_compaction_metrics(&self, _tenant_id: &TenantId) -> Result<CompactionMetrics> {
Err(MemdError::StorageError(
"compaction metrics not available".into(),
))
}
}
pub use dense::{DenseSearchConfig, DenseSearchResult, DenseSearcher};
pub use hybrid::{HybridConfig, HybridSearchResult, HybridSearcher, HybridTiming, SearchContext};
pub use memory::MemoryStore;
pub use persistent::{PersistentStore, PersistentStoreConfig, TieredStats};
pub use shared_add::{split_for_add, ADD_CHUNK_THRESHOLD};
pub use tenant::TenantManager;
pub use tombstone::TombstoneSet;