use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::IntoResponse;
use serde::{Deserialize, Serialize};
use super::bad_request;
use crate::AppState;
use crate::ingest::{IngestStats, run_ingest};
#[derive(Deserialize, Default)]
pub(super) struct IngestRequest {
source: Option<String>,
}
#[derive(Serialize, Default)]
struct IngestResponse {
new_memories: usize,
new_entities: usize,
new_conversations: usize,
skipped_dupes: usize,
unchanged: usize,
errors: usize,
files_processed: usize,
#[serde(skip_serializing_if = "std::ops::Not::not")]
deferred: bool,
}
impl IngestResponse {
fn deferred() -> Self {
Self {
deferred: true,
..Self::default()
}
}
}
impl From<&IngestStats> for IngestResponse {
fn from(stats: &IngestStats) -> Self {
Self {
new_memories: stats.new_memories,
new_entities: stats.new_entities,
new_conversations: stats.new_conversations,
skipped_dupes: stats.skipped_dupes,
unchanged: stats.unchanged,
errors: stats.errors,
files_processed: stats.files_processed,
deferred: false,
}
}
}
#[derive(Serialize)]
struct ReingestResponse {
cleared_log: usize,
#[serde(flatten)]
stats: IngestResponse,
}
pub(super) async fn ingest(
State(state): State<Arc<AppState>>,
body: Option<Json<IngestRequest>>,
) -> impl IntoResponse {
let source = body.and_then(|Json(req)| req.source);
if let Err(message) = crate::ingest::adapters_for(source.as_deref(), &state.machine.id) {
return bad_request(message);
}
if state.bootstrap.is_pending() {
return Json(IngestResponse::deferred()).into_response();
}
let store = state.store.clone();
let embedder = state.embedder.clone();
let machine_id = state.machine.id.clone();
let result = tokio::task::spawn_blocking(move || {
run_ingest(&store, embedder.as_deref(), &machine_id, source.as_deref())
})
.await;
match result {
Ok(Ok(stats)) => Json(IngestResponse::from(&stats)).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[derive(Deserialize)]
pub(super) struct ReingestRequest {
#[serde(default)]
clean: bool,
}
pub(super) async fn reingest(
State(state): State<Arc<AppState>>,
Json(req): Json<ReingestRequest>,
) -> impl IntoResponse {
if req.clean {
return bad_request(
"clean is no longer supported: raw ids are derived from the transcript, so \
deleting raws and re-ingesting would skip every chunk and leave the raws \
deleted on every peer"
.to_string(),
);
}
if state.bootstrap.is_pending() {
return Json(ReingestResponse {
cleared_log: 0,
stats: IngestResponse::deferred(),
})
.into_response();
}
let store = state.store.clone();
let embedder = state.embedder.clone();
let machine_id = state.machine.id.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<ReingestResponse> {
let (cleared_log, stats) =
crate::ingest::reingest(&store, embedder.as_deref(), &machine_id)?;
Ok(ReingestResponse {
cleared_log,
stats: IngestResponse::from(&stats),
})
})
.await;
match result {
Ok(Ok(response)) => Json(response).into_response(),
Ok(Err(e)) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
.into_response(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_answer_that_ran_carries_no_deferred_field() {
let ran = serde_json::to_value(IngestResponse::from(&IngestStats {
new_memories: 2,
..IngestStats::default()
}))
.unwrap();
assert!(ran.get("deferred").is_none(), "{ran}");
assert_eq!(ran["new_memories"], 2);
let deferred = serde_json::to_value(IngestResponse::deferred()).unwrap();
assert_eq!(deferred["deferred"], true);
}
}