use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Duration;
use age::x25519::Recipient;
use axum::Json;
use axum::body::{Body, Bytes};
use axum::extract::{Request, State};
use axum::http::request::Parts;
use axum::http::{HeaderValue, Method, StatusCode, header};
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use second_brain_core::redact::redact;
use serde_json::{Value, json};
use tokio::sync::Notify;
use crate::AppState;
use crate::clock_gate::ClockGate;
use crate::write_queue::{self, Entry};
pub const REPLAY_HEADER: &str = "x-second-brain-replay";
pub const QUEUED_REASON: &str = "store maintenance is running; the write is saved when it finishes";
const RETRY_AFTER_SECS: u64 = 300;
pub const CLOCK_HELD_REASON: &str =
"clock not yet trusted: writes are held until time sync is confirmed; retry later";
const CLOCK_RETRY_AFTER_SECS: u64 = 30;
const MAX_QUEUED_BODY: usize = 8 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Mode {
Open,
Closed,
Draining,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Class {
Read,
Queueable(&'static str),
Refused,
}
pub struct WriteGate {
queue_dir: Option<PathBuf>,
recipient: Option<Recipient>,
mode: Mutex<Mode>,
in_flight: AtomicUsize,
maintenance: Mutex<()>,
reopened: Notify,
}
impl Default for WriteGate {
fn default() -> Self {
Self::with_queue(None)
}
}
pub struct WriteSlot<'a>(&'a WriteGate);
impl Drop for WriteSlot<'_> {
fn drop(&mut self) {
self.0.in_flight.fetch_sub(1, Ordering::SeqCst);
}
}
pub struct Closed<'a> {
gate: &'a WriteGate,
_serial: MutexGuard<'a, ()>,
}
impl Drop for Closed<'_> {
fn drop(&mut self) {
*self.gate.mode() = Mode::Draining;
self.gate.reopened.notify_one();
}
}
enum Admission<'a> {
Live(WriteSlot<'a>),
Queue,
Busy,
}
impl WriteGate {
pub fn new(queue_dir: PathBuf) -> Self {
Self::with_queue(Some(queue_dir))
}
fn with_queue(queue_dir: Option<PathBuf>) -> Self {
Self {
queue_dir,
recipient: None,
mode: Mutex::new(Mode::Open),
in_flight: AtomicUsize::new(0),
maintenance: Mutex::new(()),
reopened: Notify::new(),
}
}
pub fn sealing_to(mut self, recipient: Recipient) -> Self {
self.recipient = Some(recipient);
self
}
pub fn is_closed(&self) -> bool {
*self.mode() == Mode::Closed
}
pub fn try_enter(&self) -> Option<WriteSlot<'_>> {
match self.admit(Class::Refused, true) {
Admission::Live(slot) => Some(slot),
Admission::Queue | Admission::Busy => None,
}
}
pub fn close(&self) -> Closed<'_> {
let serial = match self.maintenance.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*self.mode() = Mode::Closed;
while self.in_flight.load(Ordering::SeqCst) > 0 {
std::thread::sleep(Duration::from_millis(20));
}
Closed {
gate: self,
_serial: serial,
}
}
pub async fn reopened(&self) {
self.reopened.notified().await;
}
pub fn open_when_drained(&self, queue_empty: impl FnOnce() -> bool) {
let mut mode = self.mode();
if *mode == Mode::Draining && queue_empty() {
*mode = Mode::Open;
}
}
fn admit(&self, class: Class, replay: bool) -> Admission<'_> {
let mode = self.mode();
let admission = match (*mode, class) {
(Mode::Open, _) => Admission::Live(WriteSlot(self)),
(Mode::Closed, Class::Queueable(_)) if !replay => Admission::Queue,
(Mode::Closed, _) => Admission::Busy,
(Mode::Draining, Class::Queueable(_)) if !replay => Admission::Queue,
(Mode::Draining, _) => Admission::Live(WriteSlot(self)),
};
if matches!(admission, Admission::Live(_)) {
self.in_flight.fetch_add(1, Ordering::SeqCst);
}
admission
}
fn enqueue(&self, entry: &Entry) -> anyhow::Result<bool> {
let mode = self.mode();
if *mode == Mode::Open {
return Ok(false);
}
let Some(dir) = &self.queue_dir else {
anyhow::bail!("this daemon has no write queue");
};
write_queue::push(dir, entry, self.recipient.as_ref())?;
Ok(true)
}
fn mode(&self) -> MutexGuard<'_, Mode> {
match self.mode.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
}
}
}
fn classify(method: &Method, path: &str) -> Class {
if method == Method::GET || method == Method::HEAD {
return Class::Read;
}
match (method.as_str(), path) {
(
"POST",
"/v1/recall"
| "/v1/context"
| "/v1/daily/memories"
| "/v1/sync/export"
| "/v1/sync/snapshot"
| "/v1/sync/snapshot/apply"
| "/v1/maintenance/vacuum",
) => Class::Read,
("POST", "/v1/remember") => Class::Queueable("remember"),
("POST", "/v1/notes") => Class::Queueable("note"),
("POST", "/v1/associate") => Class::Queueable("associate"),
("POST", "/v1/correct") => Class::Queueable("correct"),
("POST", "/v1/ingest") => Class::Queueable("ingest"),
("DELETE", p) if p.starts_with("/v1/memory/") => Class::Queueable("forget"),
_ => Class::Refused,
}
}
pub async fn hold_writes(State(state): State<Arc<AppState>>, req: Request, next: Next) -> Response {
let class = classify(req.method(), req.uri().path());
if class == Class::Read {
return next.run(req).await;
}
let exempt = passes_closed_clock(req.uri().path());
if !exempt && !state.clock.is_open() {
return clock_held();
}
let replay = req.headers().contains_key(REPLAY_HEADER);
let mut req = req;
loop {
match state.gate.admit(class, replay) {
Admission::Live(slot) => {
let response = next.run(req).await;
drop(slot);
return unless_clock_closed(&state.clock, exempt, response);
}
Admission::Busy => return busy(),
Admission::Queue => {}
}
let Class::Queueable(tool) = class else {
return busy();
};
let (parts, body) = req.into_parts();
let bytes = match axum::body::to_bytes(body, MAX_QUEUED_BODY).await {
Ok(b) => b,
Err(e) => return error(StatusCode::PAYLOAD_TOO_LARGE, &format!("{e}")),
};
let entry = match queued_entry(tool, &parts, &bytes) {
Ok(entry) => entry,
Err(e) => return error(StatusCode::BAD_REQUEST, &e),
};
let saved = {
let (state, entry) = (state.clone(), entry.clone());
tokio::task::spawn_blocking(move || state.gate.enqueue(&entry)).await
};
match saved {
Ok(Ok(true)) => return queued(&entry),
Ok(Ok(false)) => req = Request::from_parts(parts, Body::from(bytes)),
Ok(Err(e)) => {
tracing::warn!("cannot queue a {tool} during store maintenance: {e:#}");
return busy();
}
Err(e) => {
tracing::warn!("queueing a {tool} during store maintenance failed: {e}");
return busy();
}
}
}
}
fn queued_entry(tool: &str, parts: &Parts, bytes: &Bytes) -> Result<Entry, String> {
let mut body: Option<Value> = if bytes.is_empty() {
None
} else {
Some(serde_json::from_slice(bytes).map_err(|e| format!("invalid JSON body: {e}"))?)
};
if names_its_write(tool) {
let Some(Value::Object(fields)) = body.as_mut() else {
return Err(format!("a {tool} needs a JSON object body"));
};
if !fields.get("id").is_some_and(Value::is_string) {
fields.insert(
"id".to_string(),
Value::String(uuid::Uuid::new_v4().to_string()),
);
}
}
if let Some(body) = body.as_mut() {
redact_strings(body);
}
Ok(Entry {
tool: tool.to_string(),
method: parts.method.to_string(),
path: parts.uri.path().to_string(),
query: parts.uri.query().map(redact_query),
body,
})
}
fn names_its_write(tool: &str) -> bool {
matches!(tool, "remember" | "note" | "correct")
}
fn redact_strings(value: &mut Value) {
match value {
Value::String(s) => *s = redact(s).text,
Value::Array(items) => items.iter_mut().for_each(redact_strings),
Value::Object(fields) => fields.values_mut().for_each(redact_strings),
_ => {}
}
}
fn redact_query(query: &str) -> String {
let Ok(mut url) = reqwest::Url::parse(&format!("http://queue/?{query}")) else {
return redact(query).text;
};
let pairs: Vec<(String, String)> = url
.query_pairs()
.map(|(k, v)| (k.into_owned(), redact(&v).text))
.collect();
url.query_pairs_mut().clear().extend_pairs(pairs);
url.query().unwrap_or_default().to_string()
}
fn queued(entry: &Entry) -> Response {
let mut reply = json!({ "queued": true, "reason": QUEUED_REASON });
if names_its_write(&entry.tool)
&& let Some(id) = entry.body.as_ref().and_then(|b| b["id"].as_str())
{
reply["id"] = Value::String(id.to_string());
}
(StatusCode::ACCEPTED, Json(reply)).into_response()
}
fn busy() -> Response {
let mut response = error(
StatusCode::SERVICE_UNAVAILABLE,
"store maintenance is running; retry later",
);
response
.headers_mut()
.insert(header::RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECS));
response
}
fn passes_closed_clock(path: &str) -> bool {
matches!(
path,
"/v1/sync/import"
| "/v1/sync/state"
| "/v1/sync/hello"
| "/v1/sync/accept-hello"
| "/v1/sync/ack"
| "/v1/sync/peer-in"
| "/v1/sync/push-report"
)
}
pub(crate) fn clock_held() -> Response {
let mut response = error(StatusCode::SERVICE_UNAVAILABLE, CLOCK_HELD_REASON);
response.headers_mut().insert(
header::RETRY_AFTER,
HeaderValue::from(CLOCK_RETRY_AFTER_SECS),
);
response
}
fn unless_clock_closed(clock: &ClockGate, exempt: bool, response: Response) -> Response {
if !exempt && response.status() == StatusCode::INTERNAL_SERVER_ERROR && !clock.is_open() {
clock_held()
} else {
response
}
}
fn error(status: StatusCode, message: &str) -> Response {
(status, Json(json!({ "error": message }))).into_response()
}
#[cfg(test)]
mod tests {
use second_brain_core::sqlite_store::clock::ClockUntrusted;
use super::*;
#[test]
fn writes_are_told_apart_from_reads() {
let post = Method::POST;
assert_eq!(classify(&Method::GET, "/v1/redaction"), Class::Read);
assert_eq!(classify(&post, "/v1/recall"), Class::Read);
assert_eq!(classify(&post, "/v1/maintenance/vacuum"), Class::Read);
assert_eq!(
classify(&post, "/v1/remember"),
Class::Queueable("remember")
);
assert_eq!(classify(&post, "/v1/notes"), Class::Queueable("note"));
assert_eq!(classify(&Method::GET, "/v1/notes"), Class::Read);
assert_eq!(
classify(&Method::DELETE, "/v1/memory/x"),
Class::Queueable("forget")
);
assert_eq!(classify(&post, "/v1/correct"), Class::Queueable("correct"));
assert_eq!(classify(&post, "/v1/sync/import"), Class::Refused);
assert_eq!(classify(&post, "/v1/some-new-route"), Class::Refused);
}
#[test]
fn a_queued_write_holds_no_secret() {
let token = ["gh", "p_"].concat() + "Qq7k3Jd9XwL2pVb8NzR4tYc6MhF1sGa5UeT0";
let remember = axum::http::Request::builder()
.method("POST")
.uri("/v1/remember")
.body(())
.unwrap()
.into_parts()
.0;
let body = json!({"content": format!("deploy with {token}"), "tags": [token.clone()]});
let entry = queued_entry("remember", &remember, &Bytes::from(body.to_string())).unwrap();
let forget = axum::http::Request::builder()
.method("DELETE")
.uri(format!("/v1/memory/x?hard=false&reason=leaked%20{token}"))
.body(())
.unwrap()
.into_parts()
.0;
let forget = queued_entry("forget", &forget, &Bytes::new()).unwrap();
let saved =
serde_json::to_string(&entry).unwrap() + &serde_json::to_string(&forget).unwrap();
assert!(!saved.contains(&token), "{saved}");
assert!(saved.contains("[REDACTED:github_token]"));
assert!(entry.body.unwrap()["id"].is_string());
assert!(forget.query.unwrap().starts_with("hard=false&reason="));
}
#[test]
fn a_queued_note_is_named_before_it_is_stored() {
let notes = axum::http::Request::builder()
.method("POST")
.uri("/v1/notes")
.body(())
.unwrap()
.into_parts()
.0;
let body = json!({"session_id": "s", "decision": "d"});
let entry = queued_entry("note", ¬es, &Bytes::from(body.to_string())).unwrap();
let id = entry.body.as_ref().unwrap()["id"]
.as_str()
.unwrap()
.to_string();
assert_eq!(queued(&entry).status(), StatusCode::ACCEPTED);
let again = queued_entry(
"note",
¬es,
&Bytes::from(entry.body.unwrap().to_string()),
);
assert_eq!(again.unwrap().body.unwrap()["id"], id.as_str());
}
#[test]
fn a_queued_correct_gets_a_client_id_it_reports() {
let parts = axum::http::Request::builder()
.method("POST")
.uri("/v1/correct")
.body(())
.unwrap()
.into_parts()
.0;
let body = json!({"memory_id": "m", "content": "fixed", "evidence": "src/lib.rs:4"});
let entry = queued_entry("correct", &parts, &Bytes::from(body.to_string())).unwrap();
let id = entry.body.as_ref().unwrap()["id"]
.as_str()
.unwrap()
.to_string();
assert!(uuid::Uuid::parse_str(&id).is_ok());
let reply = queued(&entry);
assert_eq!(reply.status(), StatusCode::ACCEPTED);
}
struct NeverSynced;
impl crate::clock_gate::TimeSignals for NeverSynced {
fn now_micros(&self) -> i64 {
0
}
fn ntp_synced(&self) -> bool {
false
}
fn rtc_secs(&self) -> Option<i64> {
None
}
}
fn closed_clock(dir: &std::path::Path) -> Arc<ClockGate> {
ClockGate::new(dir, Arc::new(NeverSynced), i64::MAX)
}
fn retry_after(response: &Response) -> Option<&str> {
response
.headers()
.get(header::RETRY_AFTER)
.map(|v| v.to_str().unwrap())
}
#[test]
fn only_sync_routes_that_mint_nothing_or_carry_a_hello_pass_a_closed_clock() {
for path in [
"/v1/sync/import",
"/v1/sync/state",
"/v1/sync/hello",
"/v1/sync/accept-hello",
"/v1/sync/ack",
"/v1/sync/peer-in",
"/v1/sync/push-report",
] {
assert!(passes_closed_clock(path), "{path}");
}
for path in [
"/v1/remember",
"/v1/ingest",
"/v1/memory/x",
"/v1/sync/compact",
"/v1/sync/prune-relations",
"/v1/maintenance/repair",
] {
assert!(!passes_closed_clock(path), "{path}");
}
}
#[test]
fn a_held_write_is_told_to_retry_in_thirty_seconds() {
let held = clock_held();
assert_eq!(held.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(retry_after(&held), Some("30"));
}
#[test]
fn a_server_error_while_the_clock_is_closed_is_answered_as_a_held_write() {
let dir = tempfile::tempdir().unwrap();
let closed = closed_clock(dir.path());
assert!(!closed.is_open());
let failed = || error(StatusCode::INTERNAL_SERVER_ERROR, "store write failed");
let held = unless_clock_closed(&closed, false, failed());
assert_eq!(held.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(retry_after(&held), Some("30"));
let refused = unless_clock_closed(&closed, false, error(StatusCode::BAD_REQUEST, "bad"));
assert_eq!(refused.status(), StatusCode::BAD_REQUEST);
let open = ClockGate::open_for_tests();
let failed = unless_clock_closed(&open, false, failed());
assert_eq!(failed.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[test]
fn a_server_error_on_a_sync_route_that_passes_a_closed_clock_stays_a_server_error() {
let dir = tempfile::tempdir().unwrap();
let closed = closed_clock(dir.path());
let failed = error(StatusCode::INTERNAL_SERVER_ERROR, "disk I/O error");
let response = unless_clock_closed(&closed, passes_closed_clock("/v1/sync/import"), failed);
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(retry_after(&response), None);
}
#[test]
fn a_handler_error_carrying_clock_untrusted_is_a_held_write() {
let untrusted = anyhow::Error::new(ClockUntrusted).context("storing a memory");
let held = crate::routes::internal_error(untrusted);
assert_eq!(held.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(retry_after(&held), Some("30"));
let other = crate::routes::internal_error(anyhow::anyhow!("database is locked"));
assert_eq!(other.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[test]
fn closing_waits_for_writes_in_flight() {
let gate = Arc::new(WriteGate::default());
let slot_gate = gate.clone();
let (entered, wait) = std::sync::mpsc::channel();
let writer = std::thread::spawn(move || {
let _slot = slot_gate.try_enter().unwrap();
entered.send(()).unwrap();
std::thread::sleep(Duration::from_millis(200));
});
wait.recv().unwrap();
let started = std::time::Instant::now();
let closed = gate.close();
assert!(started.elapsed() >= Duration::from_millis(100));
assert!(gate.try_enter().is_none());
writer.join().unwrap();
drop(closed);
assert!(gate.try_enter().is_some());
}
#[test]
fn after_maintenance_writes_queue_until_the_queue_drains() {
let gate = WriteGate::default();
drop(gate.close());
assert!(matches!(
gate.admit(Class::Queueable("remember"), false),
Admission::Queue
));
assert!(matches!(
gate.admit(Class::Queueable("remember"), true),
Admission::Live(_)
));
gate.open_when_drained(|| false);
assert!(matches!(
gate.admit(Class::Queueable("remember"), false),
Admission::Queue
));
gate.open_when_drained(|| true);
assert!(matches!(
gate.admit(Class::Queueable("remember"), false),
Admission::Live(_)
));
}
}