use std::collections::{HashMap, HashSet};
use anyhow::Result;
use chrono::{DateTime, Utc};
use rusqlite::{OptionalExtension, params};
use serde::Serialize;
use uuid::Uuid;
use crate::index::{EMBEDDING_DIM, Indexes};
use crate::retrieval::cosine;
use crate::schema::{Memory, MemoryType, Relation, RelationType, Tier};
use crate::session_dates::latest_session;
use crate::sqlite_store::{MEMORY_COLS, SqliteStore, parse_uuid, row_to_memory};
use crate::store::Store;
pub const REVIEW_CONTEXT: &str = "near-duplicate-review";
pub const UNMERGED_CONTEXT: &str = "near-duplicate-review:unmerged";
pub const MAX_GROUP: usize = 4;
const NEIGHBOURS: usize = 10;
const DISTILLATE: &str = "source = 'consolidation' AND tier = 'hot' AND superseded_by IS NULL";
#[derive(Debug, Clone, Serialize)]
pub struct NearDupMember {
pub id: Uuid,
pub content: String,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize)]
pub struct NearDupPair {
pub a: Uuid,
pub b: Uuid,
pub similarity: f32,
}
#[derive(Debug, Clone, Serialize)]
pub struct NearDupGroup {
pub project_path: Option<String>,
pub memory_type: MemoryType,
pub members: Vec<NearDupMember>,
pub pairs: Vec<NearDupPair>,
}
#[derive(Debug, Clone, Serialize)]
pub struct NearDupListing {
pub groups: Vec<NearDupGroup>,
pub total: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReviewVerdict {
Separate,
Conflict,
Unmerged,
}
impl ReviewVerdict {
fn relation(self) -> (RelationType, &'static str, &'static str) {
match self {
ReviewVerdict::Separate => (RelationType::RelatesTo, "relates_to", REVIEW_CONTEXT),
ReviewVerdict::Conflict => (RelationType::Contradicts, "contradicts", REVIEW_CONTEXT),
ReviewVerdict::Unmerged => (RelationType::RelatesTo, "relates_to", UNMERGED_CONTEXT),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum MergeRefusal {
Stale(Uuid),
Invalid(String),
}
#[derive(Debug, Clone, Serialize)]
pub struct MergeReport {
pub merged_id: Uuid,
pub superseded: Vec<Uuid>,
pub markers_repointed: u64,
}
struct Candidate {
id: Uuid,
project_path: Option<String>,
memory_type: MemoryType,
created_at: DateTime<Utc>,
embedding: Vec<f32>,
}
type Grouping = (Vec<usize>, Vec<(usize, usize, f32)>);
impl SqliteStore {
pub fn near_duplicate_groups(
&self,
indexes: &Indexes,
threshold: f32,
limit: usize,
) -> Result<NearDupListing> {
let (candidates, groups) = self.group_near_duplicates(indexes, threshold)?;
let total = groups.len();
let groups: Vec<Grouping> = groups.into_iter().take(limit).collect();
let mut out = Vec::with_capacity(groups.len());
for (members, pairs) in groups {
let first = &candidates[members[0]];
let mut listed = Vec::with_capacity(members.len());
for &i in &members {
listed.push(NearDupMember {
id: candidates[i].id,
content: self.content_of(candidates[i].id)?,
created_at: candidates[i].created_at,
});
}
out.push(NearDupGroup {
project_path: first.project_path.clone(),
memory_type: first.memory_type,
members: listed,
pairs: pairs
.into_iter()
.map(|(a, b, similarity)| NearDupPair {
a: candidates[a].id,
b: candidates[b].id,
similarity,
})
.collect(),
});
}
Ok(NearDupListing { groups: out, total })
}
pub fn near_duplicate_group_count(&self, indexes: &Indexes, threshold: f32) -> Result<usize> {
Ok(self.group_near_duplicates(indexes, threshold)?.1.len())
}
pub fn check_merge(&self, member_ids: &[Uuid]) -> Result<Result<(), MergeRefusal>> {
if let Err(refusal) = check_member_ids(member_ids) {
return Ok(Err(refusal));
}
let mut found = Vec::with_capacity(member_ids.len());
for &id in member_ids {
found.push((id, self.get_memory(id)?));
}
Ok(merge_members(found).map(|_| ()))
}
pub fn merge_distillates(
&self,
member_ids: &[Uuid],
content: &str,
embedding: Vec<f32>,
model: &str,
) -> Result<Result<MergeReport, MergeRefusal>> {
if let Err(refusal) = check_member_ids(member_ids) {
return Ok(Err(refusal));
}
if content.trim().is_empty() {
return Ok(Err(MergeRefusal::Invalid("content is empty".into())));
}
if embedding.len() != EMBEDDING_DIM {
return Ok(Err(MergeRefusal::Invalid(format!(
"embedding has {} dimensions, expected {EMBEDDING_DIM}",
embedding.len()
))));
}
self.with_txn(|tx| {
let mut found = Vec::with_capacity(member_ids.len());
for &id in member_ids {
found.push((id, Self::memory_on_tx(tx, id)?));
}
let members = match merge_members(found) {
Ok(members) => members,
Err(refusal) => return Ok(Err(refusal)),
};
let first = &members[0];
let newest = members
.iter()
.max_by_key(|m| m.created_at)
.expect("check_member_ids guarantees two members");
let mut merged = Memory::new(
content.to_string(),
first.memory_type,
"consolidation".to_string(),
newest.source_id.clone(),
)
.with_project_path(first.project_path.clone())
.with_machine_id(self.machine_id().to_string());
merged.created_at = newest.created_at;
merged.session_at = latest_session(&members);
merged.confidence = members
.iter()
.map(|m| m.confidence)
.fold(f32::MIN, f32::max);
merged.embedding = embedding;
self.store_memory_on_tx(tx, &merged)?;
let mut superseded = Vec::with_capacity(members.len());
for m in &members {
self.supersede_on_tx(tx, m.id, merged.id)?;
superseded.push(m.id);
}
let raws: Vec<Uuid> = {
let placeholders = vec!["?"; member_ids.len()].join(",");
let mut stmt = tx.prepare(&format!(
"SELECT memory_id FROM distillation_log \
WHERE distilled_id IN ({placeholders}) AND memory_id <> distilled_id \
ORDER BY memory_id"
))?;
let ids: Vec<String> = member_ids.iter().map(Uuid::to_string).collect();
let rows = stmt.query_map(rusqlite::params_from_iter(ids), |r| {
parse_uuid(&r.get::<_, String>(0)?)
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for &raw in &raws {
self.mark_consolidated_on_tx(tx, raw, merged.id, model)?;
if Self::memory_on_tx(tx, raw)?.is_some() {
self.store_relation_on_tx(
tx,
&Relation {
from_id: merged.id,
to_id: raw,
relation_type: RelationType::DistilledFrom,
strength: 1.0,
context: Some(model.to_string()),
},
)?;
}
}
Ok(Ok(MergeReport {
merged_id: merged.id,
superseded,
markers_repointed: raws.len() as u64,
}))
})
}
pub fn record_near_duplicate_review(
&self,
member_ids: &[Uuid],
verdict: ReviewVerdict,
) -> Result<Result<u64, MergeRefusal>> {
if let Err(refusal) = check_member_ids(member_ids) {
return Ok(Err(refusal));
}
let (relation_type, type_name, context) = verdict.relation();
self.with_txn(|tx| {
for &id in member_ids {
if Self::memory_on_tx(tx, id)?.is_none() {
return Ok(Err(MergeRefusal::Invalid(format!(
"no memory with id {id}"
))));
}
}
let mut sorted = member_ids.to_vec();
sorted.sort();
let mut recorded = 0u64;
for (i, &a) in sorted.iter().enumerate() {
for &b in &sorted[i + 1..] {
let exists: Option<i64> = tx
.query_row(
"SELECT 1 FROM relations WHERE from_id = ?1 AND to_id = ?2 \
AND relation_type = ?3 AND context = ?4",
params![a.to_string(), b.to_string(), type_name, context],
|r| r.get(0),
)
.optional()?;
if exists.is_some() {
continue;
}
self.store_relation_on_tx(
tx,
&Relation {
from_id: a,
to_id: b,
relation_type,
strength: 1.0,
context: Some(context.to_string()),
},
)?;
recorded += 1;
}
}
Ok(Ok(recorded))
})
}
pub fn reopen_near_duplicate_reviews(&self, verdicts: &[ReviewVerdict]) -> Result<u64> {
let mut edges = Vec::new();
{
let conn = self.reader();
for &verdict in verdicts {
let (relation_type, type_name, context) = verdict.relation();
let mut stmt = conn.prepare(
"SELECT from_id, to_id, strength FROM relations \
WHERE relation_type = ?1 AND context = ?2",
)?;
let rows = stmt.query_map(params![type_name, context], |r| {
Ok(Relation {
from_id: parse_uuid(&r.get::<_, String>(0)?)?,
to_id: parse_uuid(&r.get::<_, String>(1)?)?,
relation_type,
strength: r.get::<_, f64>(2)? as f32,
context: Some(context.to_string()),
})
})?;
for edge in rows {
edges.push(edge?);
}
}
}
self.with_txn(|tx| {
let mut removed = 0u64;
for edge in &edges {
if self.delete_relation_on_tx(tx, edge)? {
removed += 1;
}
}
Ok(removed)
})
}
fn group_near_duplicates(
&self,
indexes: &Indexes,
threshold: f32,
) -> Result<(Vec<Candidate>, Vec<Grouping>)> {
let candidates = self.live_distillates()?;
let position: HashMap<Uuid, usize> = candidates
.iter()
.enumerate()
.map(|(i, c)| (c.id, i))
.collect();
let reviewed = self.reviewed_pairs()?;
let mut edges: HashMap<(usize, usize), f32> = HashMap::new();
for (i, c) in candidates.iter().enumerate() {
for (id, _) in indexes.vector_search(self, &c.embedding, NEIGHBOURS)? {
let Some(&j) = position.get(&id) else {
continue;
};
let other = &candidates[j];
if j == i
|| other.project_path != c.project_path
|| other.memory_type != c.memory_type
{
continue;
}
let key = (i.min(j), i.max(j));
if edges.contains_key(&key) {
continue;
}
let (x, y) = (candidates[key.0].id, candidates[key.1].id);
if reviewed.contains(&(x.min(y), x.max(y))) {
continue;
}
let similarity = cosine(&c.embedding, &other.embedding);
if similarity >= threshold {
edges.insert(key, similarity);
}
}
}
let mut edges: Vec<((usize, usize), f32)> = edges.into_iter().collect();
edges.sort_by(|(ka, sa), (kb, sb)| {
sb.total_cmp(sa)
.then_with(|| candidates[ka.0].id.cmp(&candidates[kb.0].id))
.then_with(|| candidates[ka.1].id.cmp(&candidates[kb.1].id))
});
let mut sets = DisjointSets::new(candidates.len());
for &((a, b), _) in &edges {
sets.union_capped(a, b, MAX_GROUP);
}
let mut by_root: HashMap<usize, Vec<usize>> = HashMap::new();
for &((a, b), _) in &edges {
for i in [a, b] {
let members = by_root.entry(sets.find(i)).or_default();
if !members.contains(&i) {
members.push(i);
}
}
}
let mut groups: Vec<Grouping> = by_root
.into_values()
.filter(|members| members.len() > 1)
.map(|mut members| {
members.sort_by(|&x, &y| {
candidates[x]
.created_at
.cmp(&candidates[y].created_at)
.then_with(|| candidates[x].id.cmp(&candidates[y].id))
});
let pairs = edges
.iter()
.filter(|((a, b), _)| members.contains(a) && members.contains(b))
.map(|&((a, b), s)| (a, b, s))
.collect();
(members, pairs)
})
.collect();
groups.sort_by(|(ma, pa), (mb, pb)| {
pb[0]
.2
.total_cmp(&pa[0].2)
.then_with(|| candidates[ma[0]].id.cmp(&candidates[mb[0]].id))
});
Ok((candidates, groups))
}
fn live_distillates(&self) -> Result<Vec<Candidate>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT {MEMORY_COLS}, embedding FROM memories \
WHERE {DISTILLATE} AND embedding IS NOT NULL ORDER BY id"
))?;
let rows = stmt.query_map([], |r| row_to_memory(r, true))?;
let mut out = Vec::new();
for m in rows {
let m = m?;
if m.embedding.len() != EMBEDDING_DIM {
continue;
}
out.push(Candidate {
id: m.id,
project_path: m.project_path,
memory_type: m.memory_type,
created_at: m.created_at,
embedding: m.embedding,
});
}
Ok(out)
}
fn reviewed_pairs(&self) -> Result<HashSet<(Uuid, Uuid)>> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT from_id, to_id FROM relations WHERE context = ?1 OR context LIKE ?1 || ':%'",
)?;
let rows = stmt.query_map(params![REVIEW_CONTEXT], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
parse_uuid(&r.get::<_, String>(1)?)?,
))
})?;
let mut out = HashSet::new();
for pair in rows {
let (a, b) = pair?;
out.insert((a.min(b), a.max(b)));
}
Ok(out)
}
fn content_of(&self, id: Uuid) -> Result<String> {
let conn = self.reader();
Ok(conn.query_row(
"SELECT content FROM memories WHERE id = ?1",
params![id.to_string()],
|r| r.get(0),
)?)
}
}
fn merge_members(found: Vec<(Uuid, Option<Memory>)>) -> Result<Vec<Memory>, MergeRefusal> {
let mut members = Vec::with_capacity(found.len());
for (id, memory) in found {
match memory {
None => return Err(MergeRefusal::Stale(id)),
Some(m) if m.source != "consolidation" => {
return Err(MergeRefusal::Invalid(format!("{id} is not a distillate")));
}
Some(m) if m.tier != Tier::Hot || m.superseded_by.is_some() => {
return Err(MergeRefusal::Stale(id));
}
Some(m) => members.push(m),
}
}
let first = &members[0];
if members
.iter()
.any(|m| m.project_path != first.project_path || m.memory_type != first.memory_type)
{
return Err(MergeRefusal::Invalid(
"members span more than one project or memory type".into(),
));
}
Ok(members)
}
pub fn check_member_ids(member_ids: &[Uuid]) -> Result<(), MergeRefusal> {
if !(2..=MAX_GROUP).contains(&member_ids.len()) {
return Err(MergeRefusal::Invalid(format!(
"expected 2 to {MAX_GROUP} member ids, got {}",
member_ids.len()
)));
}
let distinct: HashSet<&Uuid> = member_ids.iter().collect();
if distinct.len() != member_ids.len() {
return Err(MergeRefusal::Invalid("member ids repeat".into()));
}
Ok(())
}
struct DisjointSets {
parent: Vec<usize>,
size: Vec<usize>,
}
impl DisjointSets {
fn new(n: usize) -> Self {
Self {
parent: (0..n).collect(),
size: vec![1; n],
}
}
fn find(&mut self, mut i: usize) -> usize {
while self.parent[i] != i {
self.parent[i] = self.parent[self.parent[i]];
i = self.parent[i];
}
i
}
fn union_capped(&mut self, a: usize, b: usize, cap: usize) {
let (ra, rb) = (self.find(a), self.find(b));
if ra == rb || self.size[ra] + self.size[rb] > cap {
return;
}
let (big, small) = if self.size[ra] >= self.size[rb] {
(ra, rb)
} else {
(rb, ra)
};
self.parent[small] = big;
self.size[big] += self.size[small];
}
}