Skip to main content

second_brain_api/
near_duplicates.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3
4use axum::Json;
5use axum::extract::{Query, State};
6use axum::http::StatusCode;
7use axum::response::{IntoResponse, Response};
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use uuid::Uuid;
11
12use second_brain_core::index::Indexes;
13use second_brain_core::near_duplicates::{MergeRefusal, ReviewVerdict, check_member_ids};
14use second_brain_core::redact::redact_for_store;
15use second_brain_core::sqlite_store::SqliteStore;
16
17use crate::AppState;
18use crate::routes::{internal_error, retry_write};
19
20const DEFAULT_LIMIT: usize = 20;
21const MAX_LIMIT: usize = 200;
22
23fn refuse(status: StatusCode, message: impl Into<String>) -> Response {
24    (status, Json(serde_json::json!({"error": message.into()}))).into_response()
25}
26
27fn refusal(r: MergeRefusal) -> Response {
28    match r {
29        MergeRefusal::Stale(id) => refuse(
30            StatusCode::CONFLICT,
31            format!("{id} is no longer a live distillate"),
32        ),
33        MergeRefusal::Invalid(message) => refuse(StatusCode::BAD_REQUEST, message),
34    }
35}
36
37fn parse_ids(raw: &[String]) -> Result<Vec<Uuid>, MergeRefusal> {
38    let ids = raw
39        .iter()
40        .map(|s| Uuid::parse_str(s).map_err(|_| MergeRefusal::Invalid(format!("invalid id {s:?}"))))
41        .collect::<Result<Vec<_>, _>>()?;
42    check_member_ids(&ids)?;
43    Ok(ids)
44}
45
46fn read_indexes(indexes: &std::sync::RwLock<Indexes>) -> std::sync::RwLockReadGuard<'_, Indexes> {
47    match indexes.read() {
48        Ok(g) => g,
49        Err(poisoned) => poisoned.into_inner(),
50    }
51}
52
53#[derive(Debug, Clone, Serialize)]
54pub struct NearDupCount {
55    pub groups: u64,
56    pub threshold: f32,
57    pub counted_at: DateTime<Utc>,
58}
59
60pub fn record_count(
61    cache: &std::sync::RwLock<Option<NearDupCount>>,
62    groups: usize,
63    threshold: f32,
64) {
65    let mut guard = match cache.write() {
66        Ok(g) => g,
67        Err(poisoned) => poisoned.into_inner(),
68    };
69    *guard = Some(NearDupCount {
70        groups: groups as u64,
71        threshold,
72        counted_at: Utc::now(),
73    });
74}
75
76pub fn cached_count(cache: &std::sync::RwLock<Option<NearDupCount>>) -> Option<NearDupCount> {
77    match cache.read() {
78        Ok(g) => g.clone(),
79        Err(poisoned) => poisoned.into_inner().clone(),
80    }
81}
82
83/// Recounts at `threshold` on an index handle the caller already holds; a
84/// failure only leaves the previous count in place.
85pub fn refresh_count(
86    sqlite: &SqliteStore,
87    indexes: &Indexes,
88    threshold: f32,
89    cache: &std::sync::RwLock<Option<NearDupCount>>,
90) {
91    match sqlite.near_duplicate_group_count(indexes, threshold) {
92        Ok(n) => record_count(cache, n, threshold),
93        Err(e) => tracing::warn!("near-duplicate count failed: {e:#}"),
94    }
95}
96
97pub async fn list(
98    State(state): State<Arc<AppState>>,
99    Query(params): Query<HashMap<String, String>>,
100) -> Response {
101    let limit = match params.get("limit").map(|s| s.parse::<usize>()) {
102        None => DEFAULT_LIMIT,
103        Some(Ok(n)) if (1..=MAX_LIMIT).contains(&n) => n,
104        Some(_) => {
105            return refuse(
106                StatusCode::BAD_REQUEST,
107                format!("limit must be between 1 and {MAX_LIMIT}"),
108            );
109        }
110    };
111    let threshold = match params.get("threshold").map(|s| s.parse::<f32>()) {
112        None => state.near_duplicate_threshold,
113        Some(Ok(t)) if t > 0.0 && t <= 1.0 => t,
114        Some(_) => {
115            return refuse(
116                StatusCode::BAD_REQUEST,
117                "threshold must be above 0 and at most 1",
118            );
119        }
120    };
121    let Some(indexes) = state.indexes.clone() else {
122        return refuse(
123            StatusCode::SERVICE_UNAVAILABLE,
124            "the vector index is unavailable",
125        );
126    };
127    let sqlite = state.sqlite.clone();
128    let result = tokio::task::spawn_blocking(move || {
129        let guard = read_indexes(&indexes);
130        sqlite.near_duplicate_groups(&guard, threshold, limit)
131    })
132    .await;
133    match result {
134        Ok(Ok(listing)) => {
135            if threshold == state.near_duplicate_threshold {
136                record_count(&state.near_duplicates, listing.total, threshold);
137            }
138            Json(serde_json::json!({
139                "threshold": threshold,
140                "groups": listing.groups,
141            }))
142            .into_response()
143        }
144        Ok(Err(e)) => internal_error(e),
145        Err(e) => internal_error(e),
146    }
147}
148
149#[derive(Deserialize)]
150pub struct MergeRequest {
151    member_ids: Vec<String>,
152    content: String,
153    model: String,
154}
155
156pub async fn merge(State(state): State<Arc<AppState>>, Json(req): Json<MergeRequest>) -> Response {
157    let ids = match parse_ids(&req.member_ids) {
158        Ok(ids) => ids,
159        Err(r) => return refusal(r),
160    };
161    if req.content.trim().is_empty() {
162        return refuse(StatusCode::BAD_REQUEST, "content is empty");
163    }
164    let sqlite = state.sqlite.clone();
165    let check_ids = ids.clone();
166    match tokio::task::spawn_blocking(move || sqlite.check_merge(&check_ids)).await {
167        Ok(Ok(Ok(()))) => {}
168        Ok(Ok(Err(r))) => return refusal(r),
169        Ok(Err(e)) => return internal_error(e),
170        Err(e) => return internal_error(e),
171    }
172    let Some(embedder) = state.embedder.clone() else {
173        return refuse(
174            StatusCode::SERVICE_UNAVAILABLE,
175            "the embedder is unavailable",
176        );
177    };
178    let sqlite = state.sqlite.clone();
179    let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
180        // A merged distillate has no context, so its body is its content,
181        // the same text consolidation embeds for a new distillate.
182        let content = redact_for_store(&req.content);
183        let embedding = embedder.embed(&content)?;
184        retry_write(|| sqlite.merge_distillates(&ids, &content, embedding.clone(), &req.model))
185    })
186    .await;
187    match result {
188        Ok(Ok(Ok(report))) => Json(report).into_response(),
189        Ok(Ok(Err(r))) => refusal(r),
190        Ok(Err(e)) => internal_error(e),
191        Err(e) => internal_error(e),
192    }
193}
194
195fn parse_verdict(raw: &str) -> Result<ReviewVerdict, String> {
196    match raw {
197        "separate" => Ok(ReviewVerdict::Separate),
198        "conflict" => Ok(ReviewVerdict::Conflict),
199        "unmerged" => Ok(ReviewVerdict::Unmerged),
200        other => Err(format!(
201            "unknown verdict {other:?}; valid values: separate, conflict, unmerged"
202        )),
203    }
204}
205
206#[derive(Deserialize)]
207pub struct ReviewRequest {
208    member_ids: Vec<String>,
209    verdict: String,
210}
211
212pub async fn review(
213    State(state): State<Arc<AppState>>,
214    Json(req): Json<ReviewRequest>,
215) -> Response {
216    let verdict = match parse_verdict(&req.verdict) {
217        Ok(v) => v,
218        Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
219    };
220    let ids = match parse_ids(&req.member_ids) {
221        Ok(ids) => ids,
222        Err(r) => return refusal(r),
223    };
224    let sqlite = state.sqlite.clone();
225    let result = tokio::task::spawn_blocking(move || {
226        retry_write(|| sqlite.record_near_duplicate_review(&ids, verdict))
227    })
228    .await;
229    match result {
230        Ok(Ok(Ok(recorded))) => {
231            Json(serde_json::json!({"relations_recorded": recorded})).into_response()
232        }
233        Ok(Ok(Err(r))) => refusal(r),
234        Ok(Err(e)) => internal_error(e),
235        Err(e) => internal_error(e),
236    }
237}
238
239/// Takes the body as a bare value so every malformed request is a JSON 400,
240/// not the extractor's plain-text 422.
241pub async fn reopen(
242    State(state): State<Arc<AppState>>,
243    Json(body): Json<serde_json::Value>,
244) -> Response {
245    let Some(raw) = body.get("verdicts").and_then(|v| v.as_array()) else {
246        return refuse(
247            StatusCode::BAD_REQUEST,
248            "verdicts must be a list of separate, conflict or unmerged",
249        );
250    };
251    if raw.is_empty() {
252        return refuse(StatusCode::BAD_REQUEST, "verdicts is empty");
253    }
254    let mut verdicts = Vec::with_capacity(raw.len());
255    for value in raw {
256        match parse_verdict(value.as_str().unwrap_or_default()) {
257            Ok(v) if !verdicts.contains(&v) => verdicts.push(v),
258            Ok(_) => {}
259            Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
260        }
261    }
262    let sqlite = state.sqlite.clone();
263    let result = tokio::task::spawn_blocking(move || {
264        retry_write(|| sqlite.reopen_near_duplicate_reviews(&verdicts))
265    })
266    .await;
267    match result {
268        Ok(Ok(removed)) => Json(serde_json::json!({"reviews_removed": removed})).into_response(),
269        Ok(Err(e)) => internal_error(e),
270        Err(e) => internal_error(e),
271    }
272}