1use std::collections::HashMap;
2use std::sync::Arc;
3
4use axum::Json;
5use axum::extract::{Query, State};
6use axum::http::StatusCode;
7use axum::response::{IntoResponse, Response};
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use uuid::Uuid;
11
12use second_brain_core::index::Indexes;
13use second_brain_core::near_duplicates::{MergeRefusal, ReviewVerdict, check_member_ids};
14use second_brain_core::redact::redact_for_store;
15use second_brain_core::sqlite_store::SqliteStore;
16
17use crate::AppState;
18use crate::routes::{internal_error, retry_write};
19
20const DEFAULT_LIMIT: usize = 20;
21const MAX_LIMIT: usize = 200;
22
23fn refuse(status: StatusCode, message: impl Into<String>) -> Response {
24 (status, Json(serde_json::json!({"error": message.into()}))).into_response()
25}
26
27fn refusal(r: MergeRefusal) -> Response {
28 match r {
29 MergeRefusal::Stale(id) => refuse(
30 StatusCode::CONFLICT,
31 format!("{id} is no longer a live distillate"),
32 ),
33 MergeRefusal::Invalid(message) => refuse(StatusCode::BAD_REQUEST, message),
34 }
35}
36
37fn parse_ids(raw: &[String]) -> Result<Vec<Uuid>, MergeRefusal> {
38 let ids = raw
39 .iter()
40 .map(|s| Uuid::parse_str(s).map_err(|_| MergeRefusal::Invalid(format!("invalid id {s:?}"))))
41 .collect::<Result<Vec<_>, _>>()?;
42 check_member_ids(&ids)?;
43 Ok(ids)
44}
45
46fn read_indexes(indexes: &std::sync::RwLock<Indexes>) -> std::sync::RwLockReadGuard<'_, Indexes> {
47 match indexes.read() {
48 Ok(g) => g,
49 Err(poisoned) => poisoned.into_inner(),
50 }
51}
52
53#[derive(Debug, Clone, Serialize)]
54pub struct NearDupCount {
55 pub groups: u64,
56 pub threshold: f32,
57 pub counted_at: DateTime<Utc>,
58}
59
60pub fn record_count(
61 cache: &std::sync::RwLock<Option<NearDupCount>>,
62 groups: usize,
63 threshold: f32,
64) {
65 let mut guard = match cache.write() {
66 Ok(g) => g,
67 Err(poisoned) => poisoned.into_inner(),
68 };
69 *guard = Some(NearDupCount {
70 groups: groups as u64,
71 threshold,
72 counted_at: Utc::now(),
73 });
74}
75
76pub fn cached_count(cache: &std::sync::RwLock<Option<NearDupCount>>) -> Option<NearDupCount> {
77 match cache.read() {
78 Ok(g) => g.clone(),
79 Err(poisoned) => poisoned.into_inner().clone(),
80 }
81}
82
83pub fn refresh_count(
86 sqlite: &SqliteStore,
87 indexes: &Indexes,
88 threshold: f32,
89 cache: &std::sync::RwLock<Option<NearDupCount>>,
90) {
91 match sqlite.near_duplicate_group_count(indexes, threshold) {
92 Ok(n) => record_count(cache, n, threshold),
93 Err(e) => tracing::warn!("near-duplicate count failed: {e:#}"),
94 }
95}
96
97pub async fn list(
98 State(state): State<Arc<AppState>>,
99 Query(params): Query<HashMap<String, String>>,
100) -> Response {
101 let limit = match params.get("limit").map(|s| s.parse::<usize>()) {
102 None => DEFAULT_LIMIT,
103 Some(Ok(n)) if (1..=MAX_LIMIT).contains(&n) => n,
104 Some(_) => {
105 return refuse(
106 StatusCode::BAD_REQUEST,
107 format!("limit must be between 1 and {MAX_LIMIT}"),
108 );
109 }
110 };
111 let threshold = match params.get("threshold").map(|s| s.parse::<f32>()) {
112 None => state.near_duplicate_threshold,
113 Some(Ok(t)) if t > 0.0 && t <= 1.0 => t,
114 Some(_) => {
115 return refuse(
116 StatusCode::BAD_REQUEST,
117 "threshold must be above 0 and at most 1",
118 );
119 }
120 };
121 let Some(indexes) = state.indexes.clone() else {
122 return refuse(
123 StatusCode::SERVICE_UNAVAILABLE,
124 "the vector index is unavailable",
125 );
126 };
127 let sqlite = state.sqlite.clone();
128 let result = tokio::task::spawn_blocking(move || {
129 let guard = read_indexes(&indexes);
130 sqlite.near_duplicate_groups(&guard, threshold, limit)
131 })
132 .await;
133 match result {
134 Ok(Ok(listing)) => {
135 if threshold == state.near_duplicate_threshold {
136 record_count(&state.near_duplicates, listing.total, threshold);
137 }
138 Json(serde_json::json!({
139 "threshold": threshold,
140 "groups": listing.groups,
141 }))
142 .into_response()
143 }
144 Ok(Err(e)) => internal_error(e),
145 Err(e) => internal_error(e),
146 }
147}
148
149#[derive(Deserialize)]
150pub struct MergeRequest {
151 member_ids: Vec<String>,
152 content: String,
153 model: String,
154}
155
156pub async fn merge(State(state): State<Arc<AppState>>, Json(req): Json<MergeRequest>) -> Response {
157 let ids = match parse_ids(&req.member_ids) {
158 Ok(ids) => ids,
159 Err(r) => return refusal(r),
160 };
161 if req.content.trim().is_empty() {
162 return refuse(StatusCode::BAD_REQUEST, "content is empty");
163 }
164 let sqlite = state.sqlite.clone();
165 let check_ids = ids.clone();
166 match tokio::task::spawn_blocking(move || sqlite.check_merge(&check_ids)).await {
167 Ok(Ok(Ok(()))) => {}
168 Ok(Ok(Err(r))) => return refusal(r),
169 Ok(Err(e)) => return internal_error(e),
170 Err(e) => return internal_error(e),
171 }
172 let Some(embedder) = state.embedder.clone() else {
173 return refuse(
174 StatusCode::SERVICE_UNAVAILABLE,
175 "the embedder is unavailable",
176 );
177 };
178 let sqlite = state.sqlite.clone();
179 let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
180 let content = redact_for_store(&req.content);
183 let embedding = embedder.embed(&content)?;
184 retry_write(|| sqlite.merge_distillates(&ids, &content, embedding.clone(), &req.model))
185 })
186 .await;
187 match result {
188 Ok(Ok(Ok(report))) => Json(report).into_response(),
189 Ok(Ok(Err(r))) => refusal(r),
190 Ok(Err(e)) => internal_error(e),
191 Err(e) => internal_error(e),
192 }
193}
194
195fn parse_verdict(raw: &str) -> Result<ReviewVerdict, String> {
196 match raw {
197 "separate" => Ok(ReviewVerdict::Separate),
198 "conflict" => Ok(ReviewVerdict::Conflict),
199 "unmerged" => Ok(ReviewVerdict::Unmerged),
200 other => Err(format!(
201 "unknown verdict {other:?}; valid values: separate, conflict, unmerged"
202 )),
203 }
204}
205
206#[derive(Deserialize)]
207pub struct ReviewRequest {
208 member_ids: Vec<String>,
209 verdict: String,
210}
211
212pub async fn review(
213 State(state): State<Arc<AppState>>,
214 Json(req): Json<ReviewRequest>,
215) -> Response {
216 let verdict = match parse_verdict(&req.verdict) {
217 Ok(v) => v,
218 Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
219 };
220 let ids = match parse_ids(&req.member_ids) {
221 Ok(ids) => ids,
222 Err(r) => return refusal(r),
223 };
224 let sqlite = state.sqlite.clone();
225 let result = tokio::task::spawn_blocking(move || {
226 retry_write(|| sqlite.record_near_duplicate_review(&ids, verdict))
227 })
228 .await;
229 match result {
230 Ok(Ok(Ok(recorded))) => {
231 Json(serde_json::json!({"relations_recorded": recorded})).into_response()
232 }
233 Ok(Ok(Err(r))) => refusal(r),
234 Ok(Err(e)) => internal_error(e),
235 Err(e) => internal_error(e),
236 }
237}
238
239pub async fn reopen(
242 State(state): State<Arc<AppState>>,
243 Json(body): Json<serde_json::Value>,
244) -> Response {
245 let Some(raw) = body.get("verdicts").and_then(|v| v.as_array()) else {
246 return refuse(
247 StatusCode::BAD_REQUEST,
248 "verdicts must be a list of separate, conflict or unmerged",
249 );
250 };
251 if raw.is_empty() {
252 return refuse(StatusCode::BAD_REQUEST, "verdicts is empty");
253 }
254 let mut verdicts = Vec::with_capacity(raw.len());
255 for value in raw {
256 match parse_verdict(value.as_str().unwrap_or_default()) {
257 Ok(v) if !verdicts.contains(&v) => verdicts.push(v),
258 Ok(_) => {}
259 Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
260 }
261 }
262 let sqlite = state.sqlite.clone();
263 let result = tokio::task::spawn_blocking(move || {
264 retry_write(|| sqlite.reopen_near_duplicate_reviews(&verdicts))
265 })
266 .await;
267 match result {
268 Ok(Ok(removed)) => Json(serde_json::json!({"reviews_removed": removed})).into_response(),
269 Ok(Err(e)) => internal_error(e),
270 Err(e) => internal_error(e),
271 }
272}