use std::collections::HashMap;
use std::sync::Arc;
use axum::Json;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use second_brain_core::index::Indexes;
use second_brain_core::near_duplicates::{MergeRefusal, ReviewVerdict, check_member_ids};
use second_brain_core::redact::redact_for_store;
use second_brain_core::sqlite_store::SqliteStore;
use crate::AppState;
use crate::routes::{internal_error, retry_write};
const DEFAULT_LIMIT: usize = 20;
const MAX_LIMIT: usize = 200;
fn refuse(status: StatusCode, message: impl Into<String>) -> Response {
(status, Json(serde_json::json!({"error": message.into()}))).into_response()
}
fn refusal(r: MergeRefusal) -> Response {
match r {
MergeRefusal::Stale(id) => refuse(
StatusCode::CONFLICT,
format!("{id} is no longer a live distillate"),
),
MergeRefusal::Invalid(message) => refuse(StatusCode::BAD_REQUEST, message),
}
}
fn parse_ids(raw: &[String]) -> Result<Vec<Uuid>, MergeRefusal> {
let ids = raw
.iter()
.map(|s| Uuid::parse_str(s).map_err(|_| MergeRefusal::Invalid(format!("invalid id {s:?}"))))
.collect::<Result<Vec<_>, _>>()?;
check_member_ids(&ids)?;
Ok(ids)
}
fn read_indexes(indexes: &std::sync::RwLock<Indexes>) -> std::sync::RwLockReadGuard<'_, Indexes> {
match indexes.read() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
}
}
#[derive(Debug, Clone, Serialize)]
pub struct NearDupCount {
pub groups: u64,
pub threshold: f32,
pub counted_at: DateTime<Utc>,
}
pub fn record_count(
cache: &std::sync::RwLock<Option<NearDupCount>>,
groups: usize,
threshold: f32,
) {
let mut guard = match cache.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*guard = Some(NearDupCount {
groups: groups as u64,
threshold,
counted_at: Utc::now(),
});
}
pub fn cached_count(cache: &std::sync::RwLock<Option<NearDupCount>>) -> Option<NearDupCount> {
match cache.read() {
Ok(g) => g.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
}
pub fn refresh_count(
sqlite: &SqliteStore,
indexes: &Indexes,
threshold: f32,
cache: &std::sync::RwLock<Option<NearDupCount>>,
) {
match sqlite.near_duplicate_group_count(indexes, threshold) {
Ok(n) => record_count(cache, n, threshold),
Err(e) => tracing::warn!("near-duplicate count failed: {e:#}"),
}
}
pub async fn list(
State(state): State<Arc<AppState>>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
let limit = match params.get("limit").map(|s| s.parse::<usize>()) {
None => DEFAULT_LIMIT,
Some(Ok(n)) if (1..=MAX_LIMIT).contains(&n) => n,
Some(_) => {
return refuse(
StatusCode::BAD_REQUEST,
format!("limit must be between 1 and {MAX_LIMIT}"),
);
}
};
let threshold = match params.get("threshold").map(|s| s.parse::<f32>()) {
None => state.near_duplicate_threshold,
Some(Ok(t)) if t > 0.0 && t <= 1.0 => t,
Some(_) => {
return refuse(
StatusCode::BAD_REQUEST,
"threshold must be above 0 and at most 1",
);
}
};
let Some(indexes) = state.indexes.clone() else {
return refuse(
StatusCode::SERVICE_UNAVAILABLE,
"the vector index is unavailable",
);
};
let sqlite = state.sqlite.clone();
let result = tokio::task::spawn_blocking(move || {
let guard = read_indexes(&indexes);
sqlite.near_duplicate_groups(&guard, threshold, limit)
})
.await;
match result {
Ok(Ok(listing)) => {
if threshold == state.near_duplicate_threshold {
record_count(&state.near_duplicates, listing.total, threshold);
}
Json(serde_json::json!({
"threshold": threshold,
"groups": listing.groups,
}))
.into_response()
}
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
#[derive(Deserialize)]
pub struct MergeRequest {
member_ids: Vec<String>,
content: String,
model: String,
}
pub async fn merge(State(state): State<Arc<AppState>>, Json(req): Json<MergeRequest>) -> Response {
let ids = match parse_ids(&req.member_ids) {
Ok(ids) => ids,
Err(r) => return refusal(r),
};
if req.content.trim().is_empty() {
return refuse(StatusCode::BAD_REQUEST, "content is empty");
}
let sqlite = state.sqlite.clone();
let check_ids = ids.clone();
match tokio::task::spawn_blocking(move || sqlite.check_merge(&check_ids)).await {
Ok(Ok(Ok(()))) => {}
Ok(Ok(Err(r))) => return refusal(r),
Ok(Err(e)) => return internal_error(e),
Err(e) => return internal_error(e),
}
let Some(embedder) = state.embedder.clone() else {
return refuse(
StatusCode::SERVICE_UNAVAILABLE,
"the embedder is unavailable",
);
};
let sqlite = state.sqlite.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let content = redact_for_store(&req.content);
let embedding = embedder.embed(&content)?;
retry_write(|| sqlite.merge_distillates(&ids, &content, embedding.clone(), &req.model))
})
.await;
match result {
Ok(Ok(Ok(report))) => Json(report).into_response(),
Ok(Ok(Err(r))) => refusal(r),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
fn parse_verdict(raw: &str) -> Result<ReviewVerdict, String> {
match raw {
"separate" => Ok(ReviewVerdict::Separate),
"conflict" => Ok(ReviewVerdict::Conflict),
"unmerged" => Ok(ReviewVerdict::Unmerged),
other => Err(format!(
"unknown verdict {other:?}; valid values: separate, conflict, unmerged"
)),
}
}
#[derive(Deserialize)]
pub struct ReviewRequest {
member_ids: Vec<String>,
verdict: String,
}
pub async fn review(
State(state): State<Arc<AppState>>,
Json(req): Json<ReviewRequest>,
) -> Response {
let verdict = match parse_verdict(&req.verdict) {
Ok(v) => v,
Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
};
let ids = match parse_ids(&req.member_ids) {
Ok(ids) => ids,
Err(r) => return refusal(r),
};
let sqlite = state.sqlite.clone();
let result = tokio::task::spawn_blocking(move || {
retry_write(|| sqlite.record_near_duplicate_review(&ids, verdict))
})
.await;
match result {
Ok(Ok(Ok(recorded))) => {
Json(serde_json::json!({"relations_recorded": recorded})).into_response()
}
Ok(Ok(Err(r))) => refusal(r),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
pub async fn reopen(
State(state): State<Arc<AppState>>,
Json(body): Json<serde_json::Value>,
) -> Response {
let Some(raw) = body.get("verdicts").and_then(|v| v.as_array()) else {
return refuse(
StatusCode::BAD_REQUEST,
"verdicts must be a list of separate, conflict or unmerged",
);
};
if raw.is_empty() {
return refuse(StatusCode::BAD_REQUEST, "verdicts is empty");
}
let mut verdicts = Vec::with_capacity(raw.len());
for value in raw {
match parse_verdict(value.as_str().unwrap_or_default()) {
Ok(v) if !verdicts.contains(&v) => verdicts.push(v),
Ok(_) => {}
Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
}
}
let sqlite = state.sqlite.clone();
let result = tokio::task::spawn_blocking(move || {
retry_write(|| sqlite.reopen_near_duplicate_reviews(&verdicts))
})
.await;
match result {
Ok(Ok(removed)) => Json(serde_json::json!({"reviews_removed": removed})).into_response(),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}