Skip to main content

second_brain_api/
near_duplicates.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3
4use axum::Json;
5use axum::extract::{Query, State};
6use axum::http::StatusCode;
7use axum::response::{IntoResponse, Response};
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use uuid::Uuid;
11
12use second_brain_core::index::Indexes;
13use second_brain_core::near_duplicates::{MergeRefusal, ReviewVerdict, check_member_ids};
14use second_brain_core::sqlite_store::SqliteStore;
15
16use crate::AppState;
17use crate::routes::{internal_error, retry_write};
18
19const DEFAULT_LIMIT: usize = 20;
20const MAX_LIMIT: usize = 200;
21
22fn refuse(status: StatusCode, message: impl Into<String>) -> Response {
23    (status, Json(serde_json::json!({"error": message.into()}))).into_response()
24}
25
26fn refusal(r: MergeRefusal) -> Response {
27    match r {
28        MergeRefusal::Stale(id) => refuse(
29            StatusCode::CONFLICT,
30            format!("{id} is no longer a live distillate"),
31        ),
32        MergeRefusal::Invalid(message) => refuse(StatusCode::BAD_REQUEST, message),
33    }
34}
35
36fn parse_ids(raw: &[String]) -> Result<Vec<Uuid>, MergeRefusal> {
37    let ids = raw
38        .iter()
39        .map(|s| Uuid::parse_str(s).map_err(|_| MergeRefusal::Invalid(format!("invalid id {s:?}"))))
40        .collect::<Result<Vec<_>, _>>()?;
41    check_member_ids(&ids)?;
42    Ok(ids)
43}
44
45fn read_indexes(indexes: &std::sync::RwLock<Indexes>) -> std::sync::RwLockReadGuard<'_, Indexes> {
46    match indexes.read() {
47        Ok(g) => g,
48        Err(poisoned) => poisoned.into_inner(),
49    }
50}
51
52#[derive(Debug, Clone, Serialize)]
53pub struct NearDupCount {
54    pub groups: u64,
55    pub threshold: f32,
56    pub counted_at: DateTime<Utc>,
57}
58
59pub fn record_count(
60    cache: &std::sync::RwLock<Option<NearDupCount>>,
61    groups: usize,
62    threshold: f32,
63) {
64    let mut guard = match cache.write() {
65        Ok(g) => g,
66        Err(poisoned) => poisoned.into_inner(),
67    };
68    *guard = Some(NearDupCount {
69        groups: groups as u64,
70        threshold,
71        counted_at: Utc::now(),
72    });
73}
74
75pub fn cached_count(cache: &std::sync::RwLock<Option<NearDupCount>>) -> Option<NearDupCount> {
76    match cache.read() {
77        Ok(g) => g.clone(),
78        Err(poisoned) => poisoned.into_inner().clone(),
79    }
80}
81
82/// Recounts at `threshold` on an index handle the caller already holds; a
83/// failure only leaves the previous count in place.
84pub fn refresh_count(
85    sqlite: &SqliteStore,
86    indexes: &Indexes,
87    threshold: f32,
88    cache: &std::sync::RwLock<Option<NearDupCount>>,
89) {
90    match sqlite.near_duplicate_group_count(indexes, threshold) {
91        Ok(n) => record_count(cache, n, threshold),
92        Err(e) => tracing::warn!("near-duplicate count failed: {e:#}"),
93    }
94}
95
96pub async fn list(
97    State(state): State<Arc<AppState>>,
98    Query(params): Query<HashMap<String, String>>,
99) -> Response {
100    let limit = match params.get("limit").map(|s| s.parse::<usize>()) {
101        None => DEFAULT_LIMIT,
102        Some(Ok(n)) if (1..=MAX_LIMIT).contains(&n) => n,
103        Some(_) => {
104            return refuse(
105                StatusCode::BAD_REQUEST,
106                format!("limit must be between 1 and {MAX_LIMIT}"),
107            );
108        }
109    };
110    let threshold = match params.get("threshold").map(|s| s.parse::<f32>()) {
111        None => state.near_duplicate_threshold,
112        Some(Ok(t)) if t > 0.0 && t <= 1.0 => t,
113        Some(_) => {
114            return refuse(
115                StatusCode::BAD_REQUEST,
116                "threshold must be above 0 and at most 1",
117            );
118        }
119    };
120    let Some(indexes) = state.indexes.clone() else {
121        return refuse(
122            StatusCode::SERVICE_UNAVAILABLE,
123            "the vector index is unavailable",
124        );
125    };
126    let sqlite = state.sqlite.clone();
127    let result = tokio::task::spawn_blocking(move || {
128        let guard = read_indexes(&indexes);
129        sqlite.near_duplicate_groups(&guard, threshold, limit)
130    })
131    .await;
132    match result {
133        Ok(Ok(listing)) => {
134            if threshold == state.near_duplicate_threshold {
135                record_count(&state.near_duplicates, listing.total, threshold);
136            }
137            Json(serde_json::json!({
138                "threshold": threshold,
139                "groups": listing.groups,
140            }))
141            .into_response()
142        }
143        Ok(Err(e)) => internal_error(e),
144        Err(e) => internal_error(e),
145    }
146}
147
148#[derive(Deserialize)]
149pub struct MergeRequest {
150    member_ids: Vec<String>,
151    content: String,
152    model: String,
153}
154
155pub async fn merge(State(state): State<Arc<AppState>>, Json(req): Json<MergeRequest>) -> Response {
156    let ids = match parse_ids(&req.member_ids) {
157        Ok(ids) => ids,
158        Err(r) => return refusal(r),
159    };
160    if req.content.trim().is_empty() {
161        return refuse(StatusCode::BAD_REQUEST, "content is empty");
162    }
163    let sqlite = state.sqlite.clone();
164    let check_ids = ids.clone();
165    match tokio::task::spawn_blocking(move || sqlite.check_merge(&check_ids)).await {
166        Ok(Ok(Ok(()))) => {}
167        Ok(Ok(Err(r))) => return refusal(r),
168        Ok(Err(e)) => return internal_error(e),
169        Err(e) => return internal_error(e),
170    }
171    let Some(embedder) = state.embedder.clone() else {
172        return refuse(
173            StatusCode::SERVICE_UNAVAILABLE,
174            "the embedder is unavailable",
175        );
176    };
177    let sqlite = state.sqlite.clone();
178    let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
179        // A merged distillate has no context, so its body is its content,
180        // the same text consolidation embeds for a new distillate.
181        let embedding = embedder.embed(&req.content)?;
182        retry_write(|| sqlite.merge_distillates(&ids, &req.content, embedding.clone(), &req.model))
183    })
184    .await;
185    match result {
186        Ok(Ok(Ok(report))) => Json(report).into_response(),
187        Ok(Ok(Err(r))) => refusal(r),
188        Ok(Err(e)) => internal_error(e),
189        Err(e) => internal_error(e),
190    }
191}
192
193fn parse_verdict(raw: &str) -> Result<ReviewVerdict, String> {
194    match raw {
195        "separate" => Ok(ReviewVerdict::Separate),
196        "conflict" => Ok(ReviewVerdict::Conflict),
197        "unmerged" => Ok(ReviewVerdict::Unmerged),
198        other => Err(format!(
199            "unknown verdict {other:?}; valid values: separate, conflict, unmerged"
200        )),
201    }
202}
203
204#[derive(Deserialize)]
205pub struct ReviewRequest {
206    member_ids: Vec<String>,
207    verdict: String,
208}
209
210pub async fn review(
211    State(state): State<Arc<AppState>>,
212    Json(req): Json<ReviewRequest>,
213) -> Response {
214    let verdict = match parse_verdict(&req.verdict) {
215        Ok(v) => v,
216        Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
217    };
218    let ids = match parse_ids(&req.member_ids) {
219        Ok(ids) => ids,
220        Err(r) => return refusal(r),
221    };
222    let sqlite = state.sqlite.clone();
223    let result = tokio::task::spawn_blocking(move || {
224        retry_write(|| sqlite.record_near_duplicate_review(&ids, verdict))
225    })
226    .await;
227    match result {
228        Ok(Ok(Ok(recorded))) => {
229            Json(serde_json::json!({"relations_recorded": recorded})).into_response()
230        }
231        Ok(Ok(Err(r))) => refusal(r),
232        Ok(Err(e)) => internal_error(e),
233        Err(e) => internal_error(e),
234    }
235}
236
237/// Takes the body as a bare value so every malformed request is a JSON 400,
238/// not the extractor's plain-text 422.
239pub async fn reopen(
240    State(state): State<Arc<AppState>>,
241    Json(body): Json<serde_json::Value>,
242) -> Response {
243    let Some(raw) = body.get("verdicts").and_then(|v| v.as_array()) else {
244        return refuse(
245            StatusCode::BAD_REQUEST,
246            "verdicts must be a list of separate, conflict or unmerged",
247        );
248    };
249    if raw.is_empty() {
250        return refuse(StatusCode::BAD_REQUEST, "verdicts is empty");
251    }
252    let mut verdicts = Vec::with_capacity(raw.len());
253    for value in raw {
254        match parse_verdict(value.as_str().unwrap_or_default()) {
255            Ok(v) if !verdicts.contains(&v) => verdicts.push(v),
256            Ok(_) => {}
257            Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
258        }
259    }
260    let sqlite = state.sqlite.clone();
261    let result = tokio::task::spawn_blocking(move || {
262        retry_write(|| sqlite.reopen_near_duplicate_reviews(&verdicts))
263    })
264    .await;
265    match result {
266        Ok(Ok(removed)) => Json(serde_json::json!({"reviews_removed": removed})).into_response(),
267        Ok(Err(e)) => internal_error(e),
268        Err(e) => internal_error(e),
269    }
270}