use crate::error::{MemdError, Result};
use crate::ops::{error::OperationError, DedupConfig, DedupSpec};
use crate::store::metadata::{ChunkMetadata, MetadataStore};
use crate::store::persistent::PersistentStore;
use crate::store::supersession::{canonicalize_for_type, is_near_duplicate};
use crate::types::{ChunkId, ChunkStatus, ChunkType, TenantId};
pub const DEFAULT_FUZZY_THRESHOLD: f32 = 0.92;
pub const FUZZY_RECENT_POOL_SIZE: usize = 128;
pub struct ResolvedDedup {
pub mode: DedupMode,
pub threshold: f32,
pub scope_project: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DedupMode {
Exact,
Fuzzy,
}
pub fn resolve_spec(spec: &DedupSpec) -> Result<Option<ResolvedDedup>> {
match spec {
DedupSpec::Bool(false) => Ok(None),
DedupSpec::Bool(true) => Ok(Some(ResolvedDedup {
mode: DedupMode::Exact,
threshold: DEFAULT_FUZZY_THRESHOLD,
scope_project: true,
})),
DedupSpec::Config(cfg) => resolve_config(cfg).map(Some),
}
}
fn resolve_config(cfg: &DedupConfig) -> Result<ResolvedDedup> {
let mode = match cfg.mode.as_deref().unwrap_or("exact") {
"exact" => DedupMode::Exact,
"fuzzy" => DedupMode::Fuzzy,
other => {
return Err(MemdError::ValidationError(format!(
"supersede_near_duplicates.mode: expected 'exact' or 'fuzzy', got '{other}'"
)));
}
};
let threshold = cfg.threshold.unwrap_or(DEFAULT_FUZZY_THRESHOLD);
let scope_project = match cfg.scope.as_deref().unwrap_or("project") {
"project" => true,
"tenant" => false,
other => {
return Err(MemdError::ValidationError(format!(
"supersede_near_duplicates.scope: expected 'project' or 'tenant', got '{other}'"
)));
}
};
Ok(ResolvedDedup {
mode,
threshold,
scope_project,
})
}
pub fn compute_dedup_candidates(
ps: &PersistentStore,
tenant_id: &TenantId,
text: &str,
chunk_type: ChunkType,
project_id: Option<&str>,
cfg: &ResolvedDedup,
) -> std::result::Result<Vec<ChunkId>, OperationError> {
let canonical = canonicalize_for_type(text, chunk_type);
let sql_project = if cfg.scope_project { project_id } else { None };
let candidates = match cfg.mode {
DedupMode::Exact => ps
.metadata()
.list_by_canonical_text(tenant_id, sql_project, &canonical)
.map_err(|e| OperationError::ToolError(e.to_string()))?,
DedupMode::Fuzzy => fuzzy_candidates(ps, tenant_id, &canonical, cfg, project_id)?,
};
Ok(candidates
.into_iter()
.filter(|m| project_scope_matches(m, cfg.scope_project, project_id))
.filter(is_live_head_row)
.map(|m| m.chunk_id)
.collect())
}
fn fuzzy_candidates(
ps: &PersistentStore,
tenant_id: &TenantId,
canonical: &str,
cfg: &ResolvedDedup,
project_id: Option<&str>,
) -> std::result::Result<Vec<ChunkMetadata>, OperationError> {
let metas = if cfg.scope_project && project_id.is_none() {
ps.metadata()
.list_recent_with_null_project(tenant_id, FUZZY_RECENT_POOL_SIZE)
.map_err(|e| OperationError::ToolError(e.to_string()))?
} else {
let sql_project = if cfg.scope_project { project_id } else { None };
ps.metadata()
.list_recent_for_project(tenant_id, sql_project, FUZZY_RECENT_POOL_SIZE)
.map_err(|e| OperationError::ToolError(e.to_string()))?
};
Ok(metas
.into_iter()
.filter(|m| {
is_near_duplicate(
canonical,
m.canonical_text.as_deref().unwrap_or(""),
cfg.threshold,
)
})
.collect())
}
fn project_scope_matches(
m: &ChunkMetadata,
scope_project: bool,
requested_project: Option<&str>,
) -> bool {
if !scope_project {
return true;
}
m.project_id.as_deref() == requested_project
}
fn is_live_head_row(m: &ChunkMetadata) -> bool {
m.status == ChunkStatus::Final && m.lifecycle.superseded_by.is_none()
}