use std::sync::Arc;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
use second_brain_core::retrieval::{RerankScorer, Retriever};
use second_brain_core::schema::{
EXPLICIT_SOURCES, Memory, MemoryType, Relation, RelationType, SyncOp,
};
use second_brain_core::sqlite_store::RetireOutcome;
use second_brain_core::store::{Store, StoreAdmin};
use crate::AppState;
use crate::associate::{LinkError, link};
use crate::ingest::run_ingest;
use crate::maintenance::{RepairRequest, run_repair};
pub fn router() -> Router<Arc<AppState>> {
Router::new()
.route("/health", get(health))
.route("/v1/recall", post(recall))
.route("/v1/remember", post(remember))
.route("/v1/associate", post(associate))
.route("/v1/memory/{id}", get(get_memory))
.route("/v1/memory/{id}", delete(forget))
.route("/v1/context", post(context))
.route("/v1/ingest", post(ingest))
.route("/v1/reingest", post(reingest))
.route("/v1/consolidate/batch", get(consolidate_batch))
.route("/v1/consolidate/store", post(consolidate_store))
.route("/v1/consolidate/cleanup", post(consolidate_cleanup))
.route("/v1/daily/memories", post(daily_memories))
.route("/v1/graph/traverse", get(traverse))
.route("/v1/status", get(status))
.route("/v1/health", get(health_report))
.route("/v1/maintenance/repair", post(maintenance_repair))
.route("/v1/maintenance/vacuum", post(maintenance_vacuum))
.route(
"/v1/maintenance/refile-nested-projects",
post(crate::repairs::refile_nested_projects),
)
.route(
"/v1/maintenance/restore-explicit",
post(crate::repairs::restore_explicit),
)
.route(
"/v1/maintenance/refile-stray-projects",
post(crate::repairs::refile_stray_projects),
)
.route("/v1/sync/export", post(sync_export))
.route("/v1/sync/import", post(sync_import))
.route("/v1/sync/status", get(sync_status))
.route("/v1/sync/backfill", post(sync_backfill))
.route("/v1/sync/prune-relations", post(sync_prune_relations))
.route("/v1/sync/compact", post(sync_compact))
.route("/v1/sync/state", post(set_sync_state))
.route("/v1/backfill-project-paths", post(backfill_project_paths))
.route("/v1/projects", get(list_projects))
}
#[derive(Serialize)]
struct HealthResponse {
status: &'static str,
version: &'static str,
}
async fn health() -> impl IntoResponse {
Json(HealthResponse {
status: "ok",
version: env!("CARGO_PKG_VERSION"),
})
}
const DEFAULT_RECALL_LIMIT: usize = 10;
const MAX_RECALL_LIMIT: usize = 100;
#[derive(Deserialize)]
struct RecallRequest {
query: String,
limit: Option<usize>,
source: Option<String>,
memory_type: Option<String>,
project: Option<String>,
}
#[derive(Serialize)]
struct RecallResponse {
memories: Vec<MemoryResponse>,
#[serde(skip_serializing_if = "Option::is_none")]
degraded: Option<&'static str>,
}
struct RecallOutcome {
memories: Vec<(Memory, Option<f32>)>,
degraded: Option<&'static str>,
}
#[derive(Serialize)]
struct MemoryResponse {
id: String,
content: String,
memory_type: String,
confidence: f32,
source: String,
created_at: String,
score: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
project_path: Option<String>,
machine_id: String,
machine_name: String,
}
async fn recall(
State(state): State<Arc<AppState>>,
Json(req): Json<RecallRequest>,
) -> impl IntoResponse {
let limit = req.limit.unwrap_or(DEFAULT_RECALL_LIMIT);
if !(1..=MAX_RECALL_LIMIT).contains(&limit) {
return bad_request(format!(
"limit must be between 1 and {MAX_RECALL_LIMIT}, got {limit}"
));
}
let query_text = req.query.clone();
let source_filter = req.source.clone();
let type_filter = match req.memory_type.as_deref().map(parse_memory_type) {
None => None,
Some(Ok(t)) => Some(t),
Some(Err(message)) => return bad_request(message),
};
let project_filter = req.project.clone();
let store = state.store.clone();
let sqlite = state.sqlite.clone();
let indexes = state.indexes.clone();
let reranker = state.reranker.clone();
let tunables = state.tunables;
let embedder = state.embedder.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<RecallOutcome> {
let embedding = match embedder {
Some(ref emb) => match emb.embed_query(&query_text) {
Ok(embedding) => Some(embedding),
Err(e) => {
tracing::warn!(
"embedding the recall query failed, using the text fallback: {e}"
);
None
}
},
None => None,
};
if let Some(embedding) = embedding {
let request = QueryRequest {
text: query_text.clone(),
embedding,
limit,
filters: QueryFilters {
source: source_filter.clone(),
memory_type: type_filter,
min_confidence: None,
entity_names: crate::link_query_entities(&query_text),
project_path: project_filter.clone(),
},
};
let mut results = match indexes {
Some(ref idx) => {
let retriever = Retriever { tunables };
let pool = {
let guard = match idx.read() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
retriever.candidates(&guard, &sqlite, &request)?
};
retriever.rank(
store.as_ref(),
&request,
&pool,
reranker.as_deref().map(|r| r as &dyn RerankScorer),
)?
}
None => Vec::new(),
};
if results.is_empty() {
let engine = QueryEngine::new(store.as_ref());
results = engine.recall(&request)?;
}
if !results.is_empty() {
let ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
store.record_accesses(&ids)?;
return Ok(RecallOutcome {
memories: results
.into_iter()
.map(|r| (r.memory, Some(r.score)))
.collect(),
degraded: None,
});
}
}
let text_results = sqlite.scoped_text_search(
&query_text,
source_filter.as_deref(),
type_filter,
project_filter.as_deref(),
limit,
)?;
let ids: Vec<Uuid> = text_results.iter().map(|m| m.id).collect();
store.record_accesses(&ids)?;
Ok(RecallOutcome {
memories: text_results.into_iter().map(|m| (m, None)).collect(),
degraded: Some("text-fallback"),
})
})
.await;
let outcome = match result {
Ok(Ok(outcome)) => outcome,
Ok(Err(e)) => return internal_error(e),
Err(e) => return internal_error(e),
};
let machine_names = match state.store.get_all_machines() {
Ok(names) => names,
Err(e) => return internal_error(e),
};
Json(RecallResponse {
memories: outcome
.memories
.into_iter()
.map(|(m, score)| to_response(m, score, &machine_names))
.collect(),
degraded: outcome.degraded,
})
.into_response()
}
#[derive(Deserialize)]
struct RememberRequest {
content: String,
memory_type: Option<String>,
source: Option<String>,
related_to: Option<Vec<String>>,
project: Option<String>,
}
#[derive(Serialize)]
struct RememberResponse {
id: String,
memory_type: String,
#[serde(default)]
duplicate: bool,
project_updated: bool,
entities: usize,
}
async fn remember(
State(state): State<Arc<AppState>>,
Json(req): Json<RememberRequest>,
) -> impl IntoResponse {
let memory_type = match req.memory_type.as_deref().map(parse_memory_type) {
None => MemoryType::Semantic,
Some(Ok(t)) => t,
Some(Err(message)) => return bad_request(message),
};
let source = req.source.unwrap_or_else(|| "api".to_string());
if !EXPLICIT_SOURCES.contains(&source.as_str()) {
return bad_request(format!(
"remember accepts only an explicit source ({}), not '{source}'",
EXPLICIT_SOURCES.join(", ")
));
}
let content = req.content.clone();
let related_to = req.related_to.clone();
let project = req.project.clone();
let machine_id = state.machine.id.clone();
let store = state.store.clone();
let sqlite = state.sqlite.clone();
let embedder = state.embedder.clone();
let result = tokio::task::spawn_blocking(
move || -> anyhow::Result<(Uuid, MemoryType, bool, bool, usize)> {
if let Some(existing) = sqlite.find_exact_duplicate(&content, None)? {
let project_updated = match project.as_deref() {
Some(p) => sqlite.refile_memory(existing.id, p)?,
None => false,
};
return Ok((existing.id, existing.memory_type, true, project_updated, 0));
}
let mut memory = Memory::new(content.clone(), memory_type, source, String::new())
.with_project_path(project)
.with_machine_id(machine_id);
if let Some(ref emb) = embedder
&& let Ok(embedding) = emb.embed(&content)
{
memory.embedding = embedding;
}
let memory_id = memory.id;
retry_write(|| store.store_memory(&memory))?;
let related = related_to.unwrap_or_default();
let entities = retry_write(|| {
crate::mentions::link_mentions(store.as_ref(), memory_id, &content, &related)
})?;
Ok((memory_id, memory_type, false, false, entities))
},
)
.await;
match result {
Ok(Ok((id, mt, duplicate, project_updated, entities))) => Json(RememberResponse {
id: id.to_string(),
memory_type: format!("{:?}", mt).to_lowercase(),
duplicate,
project_updated,
entities,
})
.into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct AssociateRequest {
#[serde(alias = "from")]
from_id: String,
#[serde(alias = "to")]
to_id: String,
relation: String,
context: Option<String>,
}
async fn associate(
State(state): State<Arc<AppState>>,
Json(req): Json<AssociateRequest>,
) -> impl IntoResponse {
let store = state.sqlite.clone();
let relation = req.relation.clone();
let result = tokio::task::spawn_blocking(move || {
link(&store, &req.from_id, &req.to_id, &req.relation, req.context)
})
.await;
match result {
Ok(Ok(linked)) => (
StatusCode::CREATED,
Json(serde_json::json!({
"from_id": linked.from_id,
"to_id": linked.to_id,
"relation": relation,
"superseded": linked.superseded,
})),
)
.into_response(),
Ok(Err(LinkError::BadRequest(message))) => bad_request(message),
Ok(Err(LinkError::NotFound(message))) => error_response(StatusCode::NOT_FOUND, message),
Ok(Err(LinkError::Conflict(message))) => error_response(StatusCode::CONFLICT, message),
Ok(Err(LinkError::Store(e))) => internal_error(e),
Err(e) => internal_error(e),
}
}
async fn get_memory(
State(state): State<Arc<AppState>>,
Path(id): Path<String>,
) -> impl IntoResponse {
let uuid = match Uuid::parse_str(&id) {
Ok(u) => u,
Err(_) => return (StatusCode::BAD_REQUEST, "invalid uuid").into_response(),
};
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.get_memory(uuid)).await;
let machine_names = state.store.get_all_machines().unwrap_or_default();
match result {
Ok(Ok(Some(m))) => Json(to_response(m, None, &machine_names)).into_response(),
Ok(Ok(None)) => StatusCode::NOT_FOUND.into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize, Default)]
struct ForgetParams {
#[serde(default)]
hard: bool,
reason: Option<String>,
}
const DEFAULT_FORGET_REASON: &str = "forgotten";
async fn forget(
State(state): State<Arc<AppState>>,
Path(id): Path<String>,
Query(params): Query<ForgetParams>,
) -> impl IntoResponse {
let uuid = match Uuid::parse_str(&id) {
Ok(u) => u,
Err(_) => return bad_request(format!("invalid memory id: {id}")),
};
let store = state.sqlite.clone();
let reason = params
.reason
.filter(|r| !r.trim().is_empty())
.unwrap_or_else(|| DEFAULT_FORGET_REASON.to_string());
let hard = params.hard;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<&'static str>> {
if hard {
return Ok(store.remove_memory(uuid)?.then_some("deleted"));
}
Ok(match store.retire_memory(uuid, &reason)? {
RetireOutcome::Retired => Some("retired"),
RetireOutcome::AlreadyRetired => Some("already_retired"),
RetireOutcome::NotFound => None,
})
})
.await;
match result {
Ok(Ok(Some(action))) => {
Json(serde_json::json!({"id": uuid.to_string(), "action": action})).into_response()
}
Ok(Ok(None)) => error_response(StatusCode::NOT_FOUND, format!("no memory with id {uuid}")),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
#[derive(Deserialize)]
struct ContextRequest {
project: String,
depth: Option<u32>,
}
#[derive(Serialize)]
struct ContextResponse {
memories: Vec<ContextMemory>,
}
#[derive(Serialize)]
struct ContextMemory {
id: String,
content: String,
memory_type: String,
confidence: f32,
connected: Vec<String>,
}
async fn context(
State(state): State<Arc<AppState>>,
Json(req): Json<ContextRequest>,
) -> impl IntoResponse {
let project = req.project.clone();
let depth = req.depth.unwrap_or(2);
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Vec<ContextMemory>> {
let relevant = store.memories_by_project_path(&project)?;
let mut out = Vec::new();
for memory in &relevant {
let connected: Vec<String> = store
.traverse(memory.id, depth)?
.into_iter()
.map(|(m, _)| truncate(&m.content, 100))
.collect();
out.push(ContextMemory {
id: memory.id.to_string(),
content: memory.content.clone(),
memory_type: format!("{:?}", memory.memory_type).to_lowercase(),
confidence: memory.confidence,
connected,
});
}
Ok(out)
})
.await;
match result {
Ok(Ok(memories)) => Json(ContextResponse { memories }).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize, Default)]
struct IngestRequest {
source: Option<String>,
}
#[derive(Serialize)]
struct IngestResponse {
new_memories: usize,
new_entities: usize,
new_conversations: usize,
skipped_dupes: usize,
unchanged: usize,
errors: usize,
files_processed: usize,
}
async fn ingest(
State(state): State<Arc<AppState>>,
body: Option<Json<IngestRequest>>,
) -> impl IntoResponse {
let source = body.and_then(|Json(req)| req.source);
if let Err(message) = crate::ingest::adapters_for(source.as_deref(), &state.machine.id) {
return bad_request(message);
}
let store = state.store.clone();
let embedder = state.embedder.clone();
let machine_id = state.machine.id.clone();
let result = tokio::task::spawn_blocking(move || {
run_ingest(&store, embedder.as_deref(), &machine_id, source.as_deref())
})
.await;
match result {
Ok(Ok(stats)) => Json(IngestResponse {
new_memories: stats.new_memories,
new_entities: stats.new_entities,
new_conversations: stats.new_conversations,
skipped_dupes: stats.skipped_dupes,
unchanged: stats.unchanged,
errors: stats.errors,
files_processed: stats.files_processed,
})
.into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct ReingestRequest {
#[serde(default)]
clean: bool,
}
async fn reingest(
State(state): State<Arc<AppState>>,
Json(req): Json<ReingestRequest>,
) -> impl IntoResponse {
let store = state.store.clone();
let embedder = state.embedder.clone();
let machine_id = state.machine.id.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<serde_json::Value> {
let mut cleaned_claude = 0usize;
let mut cleaned_cursor = 0usize;
if req.clean {
cleaned_claude = store.delete_memories_by_source("claude-code")?;
cleaned_cursor = store.delete_memories_by_source("cursor")?;
}
let cleared_log = store.clear_ingest_log()?;
let stats = run_ingest(&store, embedder.as_deref(), &machine_id, None)?;
if req.clean {
store.rebuild_vector_index()?;
}
Ok(serde_json::json!({
"cleaned_claude": cleaned_claude,
"cleaned_cursor": cleaned_cursor,
"cleared_log": cleared_log,
"new_memories": stats.new_memories,
"new_entities": stats.new_entities,
"new_conversations": stats.new_conversations,
"skipped_dupes": stats.skipped_dupes,
"unchanged": stats.unchanged,
"errors": stats.errors,
"files_processed": stats.files_processed,
}))
})
.await;
match result {
Ok(Ok(json)) => Json(json).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct ConsolidateBatchParams {
limit: Option<usize>,
}
async fn consolidate_batch(
State(state): State<Arc<AppState>>,
Query(params): Query<ConsolidateBatchParams>,
) -> impl IntoResponse {
let store = state.store.clone();
let limit = params.limit.unwrap_or(50);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<serde_json::Value> {
let total = store.memory_count()?;
let consolidation = store.consolidation_stats()?;
let memories = store.unconsolidated_memories(limit)?;
let mem_json: Vec<serde_json::Value> = memories
.iter()
.map(|m| {
serde_json::json!({
"id": m.id.to_string(),
"content": m.content,
"memory_type": format!("{:?}", m.memory_type).to_lowercase(),
"source": m.source,
"source_id": m.source_id,
"created_at": m.created_at.to_rfc3339(),
"project_path": m.project_path,
})
})
.collect();
Ok(serde_json::json!({
"total": total,
"consolidation": consolidation,
"memories": mem_json,
}))
})
.await;
match result {
Ok(Ok(json)) => Json(json).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct ConsolidateStoreRequest {
raw_ids: Vec<String>,
source_id: String,
#[serde(default)]
project_path: Option<String>,
distilled_content: String,
model: String,
#[serde(default)]
skip: bool,
}
async fn consolidate_store(
State(state): State<Arc<AppState>>,
Json(req): Json<ConsolidateStoreRequest>,
) -> impl IntoResponse {
let store = state.store.clone();
let embedder = state.embedder.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<serde_json::Value> {
let raw_ids: Vec<Uuid> = req
.raw_ids
.iter()
.map(|s| Uuid::parse_str(s))
.collect::<Result<_, _>>()?;
if req.skip {
for raw_id in &raw_ids {
store.mark_consolidated(*raw_id, *raw_id, &req.model)?;
}
return Ok(serde_json::json!({"status": "skipped", "count": raw_ids.len()}));
}
let mut distilled = Memory::new(
req.distilled_content.clone(),
MemoryType::Semantic,
"consolidation".to_string(),
req.source_id.clone(),
);
distilled.project_path = req.project_path.clone();
if let Some(ref emb) = embedder
&& let Ok(embedding) = emb.embed(&req.distilled_content)
{
distilled.embedding = embedding;
}
store.store_memory(&distilled)?;
for raw_id in &raw_ids {
let link = Relation {
from_id: distilled.id,
to_id: *raw_id,
relation_type: RelationType::DistilledFrom,
strength: 1.0,
context: Some(req.model.clone()),
};
store.store_relation(&link).ok();
store.mark_consolidated(*raw_id, distilled.id, &req.model)?;
}
Ok(serde_json::json!({
"status": "stored",
"distilled_id": distilled.id.to_string(),
"raw_count": raw_ids.len(),
}))
})
.await;
match result {
Ok(Ok(json)) => Json(json).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn consolidate_cleanup(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.retire_consolidated_raws()).await;
match result {
Ok(Ok(count)) => Json(serde_json::json!({"archived": count, "deleted": 0})).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct DailyMemoriesRequest {
date: String,
}
async fn daily_memories(
State(state): State<Arc<AppState>>,
Json(req): Json<DailyMemoriesRequest>,
) -> impl IntoResponse {
let date = match chrono::NaiveDate::parse_from_str(&req.date, "%Y-%m-%d") {
Ok(d) => d,
Err(_) => {
return (
StatusCode::BAD_REQUEST,
Json(serde_json::json!({"error": "invalid date, expected YYYY-MM-DD"})),
)
.into_response();
}
};
let day_start = date.and_hms_opt(0, 0, 0).unwrap().and_utc();
let day_end = (date + chrono::Duration::days(1))
.and_hms_opt(0, 0, 0)
.unwrap()
.and_utc();
let store = state.store.clone();
let result =
tokio::task::spawn_blocking(move || store.memories_created_between(&day_start, &day_end))
.await;
match result {
Ok(Ok(memories)) => {
let count = memories.len();
let items: Vec<serde_json::Value> = memories
.into_iter()
.map(|m| {
serde_json::json!({
"id": m.id.to_string(),
"content": m.content,
"memory_type": format!("{:?}", m.memory_type).to_lowercase(),
"confidence": m.confidence,
"created_at": m.created_at.to_rfc3339(),
"source": m.source,
})
})
.collect();
Json(serde_json::json!({"memories": items, "count": count})).into_response()
}
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct TraverseParams {
start_id: String,
depth: Option<u32>,
}
async fn traverse(
State(state): State<Arc<AppState>>,
Query(params): Query<TraverseParams>,
) -> impl IntoResponse {
let uuid = match Uuid::parse_str(¶ms.start_id) {
Ok(u) => u,
Err(_) => return (StatusCode::BAD_REQUEST, "invalid start_id").into_response(),
};
let depth = params.depth.unwrap_or(2);
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.traverse(uuid, depth)).await;
let machine_names = state.store.get_all_machines().unwrap_or_default();
match result {
Ok(Ok(results)) => {
let memories: Vec<MemoryResponse> = results
.into_iter()
.map(|(m, _)| to_response(m, None, &machine_names))
.collect();
Json(serde_json::json!({"memories": memories})).into_response()
}
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn status(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let sqlite = state.sqlite.clone();
let indexes = state.indexes.clone();
let embedder_available = state.embedder.is_some();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let counts = sqlite.live_counts()?;
let files = sqlite.ingested_file_count()?;
let diag = sqlite.diagnostic()?;
let consolidation = sqlite.consolidation_stats()?;
let index = index_status(&sqlite, indexes.as_deref())?;
Ok((counts, files, diag, consolidation, index))
})
.await;
let (counts, ingested_file_count, diagnostic, consolidation, index) = match result {
Ok(Ok(parts)) => parts,
Ok(Err(e)) => return internal_error(e),
Err(e) => return internal_error(e),
};
let maintenance = match state.maintenance.read() {
Ok(g) => g.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
};
let heap = crate::heap::heap_stats();
Json(serde_json::json!({
"memory_count": counts.live,
"stored_total": counts.stored_total,
"archived": counts.archived,
"superseded": counts.superseded,
"by_type": counts.by_type,
"index": index,
"ingested_file_count": ingested_file_count,
"version": env!("CARGO_PKG_VERSION"),
"embedder_available": embedder_available,
"diagnostic": diagnostic,
"machine_id": state.machine.id,
"heap": heap,
"consolidation": consolidation,
"maintenance": maintenance,
}))
.into_response()
}
fn index_status(
sqlite: &second_brain_core::sqlite_store::SqliteStore,
indexes: Option<&std::sync::RwLock<second_brain_core::index::Indexes>>,
) -> anyhow::Result<serde_json::Value> {
let Some(indexes) = indexes else {
return Ok(serde_json::json!({"verdict": "unavailable"}));
};
let pending = sqlite.outbox_pending()?;
let guard = match indexes.read() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
let verdict = if pending > 0 {
"draining"
} else if guard.consistent_with(sqlite)? {
"consistent"
} else {
"inconsistent"
};
Ok(serde_json::json!({
"verdict": verdict,
"outbox_pending": pending,
"vectors": guard.vector_len(),
"documents": guard.lexical_len(),
}))
}
pub(crate) fn internal_error(e: impl std::fmt::Display) -> axum::response::Response {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response()
}
async fn health_report(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let sqlite = state.sqlite.clone();
let indexes = state.indexes.clone();
let result = tokio::task::spawn_blocking(move || match indexes {
Some(idx) => {
let guard = match idx.read() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
sqlite.integrity_report(Some(&guard))
}
None => sqlite.integrity_report(None),
})
.await;
match result {
Ok(Ok(report)) => Json(report).into_response(),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
#[derive(Deserialize, Default)]
struct RepairBody {
#[serde(default)]
dry_run: bool,
}
async fn maintenance_repair(
State(state): State<Arc<AppState>>,
body: Option<Json<RepairBody>>,
) -> impl IntoResponse {
let dry_run = body.map(|Json(b)| b.dry_run).unwrap_or(false);
let result = tokio::task::spawn_blocking(move || match &state.repair_tx {
Some(tx) => {
let (reply_tx, reply_rx) = std::sync::mpsc::channel();
tx.send(RepairRequest {
dry_run,
reply: reply_tx,
})
.map_err(|_| anyhow::anyhow!("the index drain thread is not running"))?;
reply_rx
.recv_timeout(std::time::Duration::from_secs(3600))
.map_err(|e| anyhow::anyhow!("waiting for the drain thread: {e}"))?
}
None => {
let summary = run_repair(&state.sqlite, None, state.embedder.as_deref(), dry_run)?;
if !dry_run {
let mut guard = match state.maintenance.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*guard = Some(summary.clone());
}
Ok(summary)
}
})
.await;
match result {
Ok(Ok(summary)) => Json(summary).into_response(),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
async fn maintenance_vacuum(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let sqlite = state.sqlite.clone();
let result = tokio::task::spawn_blocking(move || sqlite.vacuum()).await;
match result {
Ok(Ok((before, after))) => Json(serde_json::json!({
"bytes_before": before,
"bytes_after": after,
}))
.into_response(),
Ok(Err(e)) => internal_error(e),
Err(e) => internal_error(e),
}
}
#[derive(Deserialize)]
struct SyncExportRequest {
after_seq: Option<u64>,
limit: Option<usize>,
}
#[derive(Serialize)]
struct SyncExportResponse {
records: Vec<serde_json::Value>,
max_seq: u64,
}
async fn sync_export(
State(state): State<Arc<AppState>>,
Json(req): Json<SyncExportRequest>,
) -> impl IntoResponse {
let after_seq = req.after_seq.unwrap_or(0);
let limit = req.limit;
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<SyncExportResponse> {
if limit.is_some() {
let capped = store.sync_log_page(after_seq, limit)?;
let mut sync_records: Vec<second_brain_sync::export::SyncRecord> = capped
.iter()
.map(|e| {
let data = e.data.as_ref().and_then(|d| serde_json::from_str(d).ok());
second_brain_sync::export::SyncRecord {
local_seq: e.local_seq as u64,
origin_machine_id: e.origin_machine_id.clone(),
origin_seq: e.origin_seq as u64,
op: e.op,
node_type: e.node_type,
node_id: e.node_id.clone(),
timestamp: e.timestamp,
data,
}
})
.collect();
second_brain_sync::export::sort_for_import(&mut sync_records);
let max_seq = sync_records
.iter()
.map(|r| r.local_seq)
.max()
.unwrap_or(after_seq);
let records: Vec<serde_json::Value> = sync_records
.iter()
.map(|r| {
serde_json::json!({
"local_seq": r.local_seq,
"origin_machine_id": r.origin_machine_id,
"origin_seq": r.origin_seq,
"op": r.op,
"node_type": r.node_type,
"node_id": r.node_id,
"timestamp": r.timestamp,
"data": r.data,
})
})
.collect();
Ok(SyncExportResponse { records, max_seq })
} else {
let mut buf = Vec::new();
let max_seq =
second_brain_sync::export::export_changes(store.as_ref(), after_seq, &mut buf)?;
let jsonl = String::from_utf8(buf)?;
let records: Vec<serde_json::Value> = jsonl
.lines()
.filter(|l| !l.trim().is_empty())
.filter_map(|l| serde_json::from_str(l).ok())
.collect();
Ok(SyncExportResponse { records, max_seq })
}
})
.await;
match result {
Ok(Ok(resp)) => Json(resp).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct SyncImportRequest {
records: Vec<serde_json::Value>,
}
#[derive(Serialize)]
struct SyncImportResponse {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
#[serde(default)]
skipped_by_reason: std::collections::BTreeMap<String, u64>,
errors: u64,
max_seq: u64,
}
async fn sync_import(
State(state): State<Arc<AppState>>,
Json(req): Json<SyncImportRequest>,
) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<SyncImportResponse> {
let mut jsonl = String::new();
for record in &req.records {
jsonl.push_str(&serde_json::to_string(record)?);
jsonl.push('\n');
}
let reader = std::io::BufReader::new(jsonl.as_bytes());
let (stats, watermark) = second_brain_sync::import::import_changes(store.as_ref(), reader)?;
Ok(SyncImportResponse {
created: stats.created,
updated: stats.updated,
deleted: stats.deleted,
skipped: stats.skipped,
skipped_by_reason: stats.skipped_by_reason,
errors: stats.errors,
max_seq: watermark,
})
})
.await;
match result {
Ok(Ok(resp)) => Json(resp).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Serialize)]
struct SyncPeerStatus {
peer_id: String,
last_seq: u64,
last_sync_at: String,
}
async fn sync_status(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.get_all_sync_states()).await;
match result {
Ok(Ok(states)) => {
let peers: Vec<SyncPeerStatus> = states
.into_iter()
.map(|s| SyncPeerStatus {
peer_id: s.peer_id,
last_seq: s.last_seq,
last_sync_at: s.last_sync_at.to_rfc3339(),
})
.collect();
Json(serde_json::json!({"peers": peers})).into_response()
}
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn sync_prune_relations(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.prune_dangling_relations()).await;
match result {
Ok(Ok(pruned)) => Json(serde_json::json!({"pruned": pruned})).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct SyncCompactRequest {
ops: Vec<SyncOp>,
#[serde(default = "default_true")]
dry_run: bool,
}
fn default_true() -> bool {
true
}
async fn sync_compact(
State(state): State<Arc<AppState>>,
Json(req): Json<SyncCompactRequest>,
) -> impl IntoResponse {
let store = state.store.clone();
let result =
tokio::task::spawn_blocking(move || store.compact_sync_log(&req.ops, req.dry_run)).await;
match result {
Ok(Ok(stats)) => Json(serde_json::json!({
"groups_with_duplicates": stats.groups_with_duplicates,
"rows_deleted": stats.rows_deleted,
"max_local_seq_before": stats.max_local_seq_before,
"max_local_seq_after": stats.max_local_seq_after,
}))
.into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn sync_backfill(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || {
let created = store.backfill_sync_log()?;
let deleted = store.backfill_tombstones()?;
anyhow::Ok((created, deleted))
})
.await;
match result {
Ok(Ok((count, tombstones))) => {
Json(serde_json::json!({"backfilled": count, "tombstones_backfilled": tombstones}))
.into_response()
}
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
struct SetSyncStateRequest {
peer_id: String,
last_seq: u64,
}
async fn set_sync_state(
State(state): State<Arc<AppState>>,
Json(req): Json<SetSyncStateRequest>,
) -> impl IntoResponse {
let store = state.store.clone();
let peer_id = req.peer_id;
let last_seq = req.last_seq;
let result = tokio::task::spawn_blocking(move || {
use second_brain_core::schema::SyncState;
store.set_sync_state(&SyncState {
peer_id,
last_seq,
last_sync_at: chrono::Utc::now(),
})
})
.await;
match result {
Ok(Ok(_)) => StatusCode::OK.into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn backfill_project_paths(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.backfill_project_paths()).await;
match result {
Ok(Ok(count)) => Json(serde_json::json!({"backfilled": count})).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
async fn list_projects(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let store = state.store.clone();
let result = tokio::task::spawn_blocking(move || store.distinct_project_paths()).await;
match result {
Ok(Ok(projects)) => {
let projects: Vec<serde_json::Value> = projects
.into_iter()
.map(|(path, count)| serde_json::json!({"project_path": path, "count": count}))
.collect();
Json(serde_json::json!({"projects": projects})).into_response()
}
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
fn to_response(
m: Memory,
score: Option<f32>,
machine_names: &std::collections::HashMap<String, String>,
) -> MemoryResponse {
let machine_name = machine_names
.get(&m.machine_id)
.cloned()
.unwrap_or_default();
MemoryResponse {
id: m.id.to_string(),
content: m.content,
memory_type: format!("{:?}", m.memory_type).to_lowercase(),
confidence: m.confidence,
source: m.source,
created_at: m.created_at.to_rfc3339(),
score,
project_path: m.project_path,
machine_id: m.machine_id,
machine_name,
}
}
const MEMORY_TYPES: [MemoryType; 8] = [
MemoryType::Episodic,
MemoryType::Semantic,
MemoryType::Procedural,
MemoryType::Decision,
MemoryType::Architecture,
MemoryType::Debugging,
MemoryType::Task,
MemoryType::Question,
];
fn memory_type_name(t: MemoryType) -> String {
format!("{t:?}").to_lowercase()
}
fn parse_memory_type(s: &str) -> Result<MemoryType, String> {
MEMORY_TYPES
.into_iter()
.find(|t| memory_type_name(*t) == s)
.ok_or_else(|| {
let valid: Vec<String> = MEMORY_TYPES.into_iter().map(memory_type_name).collect();
format!(
"unknown memory_type '{s}'; valid values: {}",
valid.join(", ")
)
})
}
fn error_response(status: StatusCode, message: impl std::fmt::Display) -> axum::response::Response {
(
status,
Json(serde_json::json!({"error": message.to_string()})),
)
.into_response()
}
fn bad_request(message: impl std::fmt::Display) -> axum::response::Response {
error_response(StatusCode::BAD_REQUEST, message)
}
fn truncate(s: &str, max_len: usize) -> String {
if s.len() <= max_len {
s.to_string()
} else {
let mut end = max_len.min(s.len());
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
format!("{}...", &s[..end])
}
}
pub(crate) fn retry_write<F, T>(mut op: F) -> anyhow::Result<T>
where
F: FnMut() -> anyhow::Result<T>,
{
let mut last_err = None;
for attempt in 0..5u32 {
match op() {
Ok(v) => return Ok(v),
Err(e) => {
last_err = Some(e);
std::thread::sleep(std::time::Duration::from_millis(50 * 2u64.pow(attempt)));
}
}
}
Err(last_err.unwrap())
}