Skip to main content

second_brain_api/
write_gate.rs

1//! Keeps agent writes off the SQLite writer while store maintenance holds
2//! it. A write that arrives then is saved to the write queue and answered
3//! 202; once maintenance ends, writes keep queueing behind the saved ones
4//! until the replayer has sent them all, so none lands out of order.
5
6use std::path::PathBuf;
7use std::sync::atomic::{AtomicUsize, Ordering};
8use std::sync::{Arc, Mutex, MutexGuard};
9use std::time::Duration;
10
11use axum::Json;
12use axum::body::{Body, Bytes};
13use axum::extract::{Request, State};
14use axum::http::request::Parts;
15use axum::http::{HeaderValue, Method, StatusCode, header};
16use axum::middleware::Next;
17use axum::response::{IntoResponse, Response};
18use second_brain_core::redact::redact;
19use serde_json::{Value, json};
20use tokio::sync::Notify;
21
22use crate::AppState;
23use crate::write_queue::{self, Entry};
24
25/// Set by the replayer, so a queued write it sends is never queued again.
26pub const REPLAY_HEADER: &str = "x-second-brain-replay";
27pub const QUEUED_REASON: &str = "store maintenance is running; the write is saved when it finishes";
28const RETRY_AFTER_SECS: u64 = 300;
29const MAX_QUEUED_BODY: usize = 8 * 1024 * 1024;
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32enum Mode {
33    Open,
34    Closed,
35    /// Maintenance ended but queued writes are still unsent.
36    Draining,
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40enum Class {
41    Read,
42    /// An agent write the replayer can send later.
43    Queueable(&'static str),
44    /// A write whose caller needs the answer now; it retries later.
45    Refused,
46}
47
48pub struct WriteGate {
49    queue_dir: Option<PathBuf>,
50    mode: Mutex<Mode>,
51    in_flight: AtomicUsize,
52    maintenance: Mutex<()>,
53    reopened: Notify,
54}
55
56impl Default for WriteGate {
57    fn default() -> Self {
58        Self::with_queue(None)
59    }
60}
61
62pub struct WriteSlot<'a>(&'a WriteGate);
63
64impl Drop for WriteSlot<'_> {
65    fn drop(&mut self) {
66        self.0.in_flight.fetch_sub(1, Ordering::SeqCst);
67    }
68}
69
70/// Held for the length of a maintenance run; dropping it lets writes back
71/// in behind whatever was queued meanwhile.
72pub struct Closed<'a> {
73    gate: &'a WriteGate,
74    _serial: MutexGuard<'a, ()>,
75}
76
77impl Drop for Closed<'_> {
78    fn drop(&mut self) {
79        *self.gate.mode() = Mode::Draining;
80        self.gate.reopened.notify_one();
81    }
82}
83
84enum Admission<'a> {
85    Live(WriteSlot<'a>),
86    Queue,
87    Busy,
88}
89
90impl WriteGate {
91    pub fn new(queue_dir: PathBuf) -> Self {
92        Self::with_queue(Some(queue_dir))
93    }
94
95    fn with_queue(queue_dir: Option<PathBuf>) -> Self {
96        Self {
97            queue_dir,
98            mode: Mutex::new(Mode::Open),
99            in_flight: AtomicUsize::new(0),
100            maintenance: Mutex::new(()),
101            reopened: Notify::new(),
102        }
103    }
104
105    pub fn is_closed(&self) -> bool {
106        *self.mode() == Mode::Closed
107    }
108
109    /// Room for an incidental write, such as recall's access count, which
110    /// is skipped rather than queued while maintenance runs.
111    pub fn try_enter(&self) -> Option<WriteSlot<'_>> {
112        match self.admit(Class::Refused, true) {
113            Admission::Live(slot) => Some(slot),
114            Admission::Queue | Admission::Busy => None,
115        }
116    }
117
118    /// Stops new writes and waits for those in flight, so the caller has the
119    /// SQLite writer to itself. Maintenance runs one at a time.
120    pub fn close(&self) -> Closed<'_> {
121        let serial = match self.maintenance.lock() {
122            Ok(g) => g,
123            Err(poisoned) => poisoned.into_inner(),
124        };
125        *self.mode() = Mode::Closed;
126        while self.in_flight.load(Ordering::SeqCst) > 0 {
127            std::thread::sleep(Duration::from_millis(20));
128        }
129        Closed {
130            gate: self,
131            _serial: serial,
132        }
133    }
134
135    /// Resolves once maintenance has ended and queued writes wait to be sent.
136    pub async fn reopened(&self) {
137        self.reopened.notified().await;
138    }
139
140    /// Lets writes through again once nothing is left queued; `queue_empty`
141    /// runs under the gate's lock, so no write can be queued after it looked.
142    pub fn open_when_drained(&self, queue_empty: impl FnOnce() -> bool) {
143        let mut mode = self.mode();
144        if *mode == Mode::Draining && queue_empty() {
145            *mode = Mode::Open;
146        }
147    }
148
149    fn admit(&self, class: Class, replay: bool) -> Admission<'_> {
150        let mode = self.mode();
151        let admission = match (*mode, class) {
152            (Mode::Open, _) => Admission::Live(WriteSlot(self)),
153            (Mode::Closed, Class::Queueable(_)) if !replay => Admission::Queue,
154            (Mode::Closed, _) => Admission::Busy,
155            (Mode::Draining, Class::Queueable(_)) if !replay => Admission::Queue,
156            (Mode::Draining, _) => Admission::Live(WriteSlot(self)),
157        };
158        if matches!(admission, Admission::Live(_)) {
159            self.in_flight.fetch_add(1, Ordering::SeqCst);
160        }
161        admission
162    }
163
164    /// Ok(false) when the gate opened since the caller was told to queue.
165    fn enqueue(&self, entry: &Entry) -> anyhow::Result<bool> {
166        let mode = self.mode();
167        if *mode == Mode::Open {
168            return Ok(false);
169        }
170        let Some(dir) = &self.queue_dir else {
171            anyhow::bail!("this daemon has no write queue");
172        };
173        write_queue::push(dir, entry)?;
174        Ok(true)
175    }
176
177    fn mode(&self) -> MutexGuard<'_, Mode> {
178        match self.mode.lock() {
179            Ok(g) => g,
180            Err(poisoned) => poisoned.into_inner(),
181        }
182    }
183}
184
185/// GET routes and the POST routes that only read pass untouched, as does
186/// the vacuum route, which closes the gate itself.
187fn classify(method: &Method, path: &str) -> Class {
188    if method == Method::GET || method == Method::HEAD {
189        return Class::Read;
190    }
191    match (method.as_str(), path) {
192        (
193            "POST",
194            "/v1/recall"
195            | "/v1/context"
196            | "/v1/daily/memories"
197            | "/v1/sync/export"
198            | "/v1/maintenance/vacuum",
199        ) => Class::Read,
200        ("POST", "/v1/remember") => Class::Queueable("remember"),
201        ("POST", "/v1/notes") => Class::Queueable("note"),
202        ("POST", "/v1/associate") => Class::Queueable("associate"),
203        ("POST", "/v1/correct") => Class::Queueable("correct"),
204        ("POST", "/v1/ingest") => Class::Queueable("ingest"),
205        ("DELETE", p) if p.starts_with("/v1/memory/") => Class::Queueable("forget"),
206        _ => Class::Refused,
207    }
208}
209
210pub async fn hold_writes(State(state): State<Arc<AppState>>, req: Request, next: Next) -> Response {
211    let class = classify(req.method(), req.uri().path());
212    if class == Class::Read {
213        return next.run(req).await;
214    }
215    let replay = req.headers().contains_key(REPLAY_HEADER);
216    let mut req = req;
217    loop {
218        match state.gate.admit(class, replay) {
219            Admission::Live(slot) => {
220                let response = next.run(req).await;
221                drop(slot);
222                return response;
223            }
224            Admission::Busy => return busy(),
225            Admission::Queue => {}
226        }
227        let Class::Queueable(tool) = class else {
228            return busy();
229        };
230        let (parts, body) = req.into_parts();
231        let bytes = match axum::body::to_bytes(body, MAX_QUEUED_BODY).await {
232            Ok(b) => b,
233            Err(e) => return error(StatusCode::PAYLOAD_TOO_LARGE, &format!("{e}")),
234        };
235        let entry = match queued_entry(tool, &parts, &bytes) {
236            Ok(entry) => entry,
237            Err(e) => return error(StatusCode::BAD_REQUEST, &e),
238        };
239        let saved = {
240            let (state, entry) = (state.clone(), entry.clone());
241            tokio::task::spawn_blocking(move || state.gate.enqueue(&entry)).await
242        };
243        match saved {
244            Ok(Ok(true)) => return queued(&entry),
245            Ok(Ok(false)) => req = Request::from_parts(parts, Body::from(bytes)),
246            Ok(Err(e)) => {
247                tracing::warn!("cannot queue a {tool} during store maintenance: {e:#}");
248                return busy();
249            }
250            Err(e) => {
251                tracing::warn!("queueing a {tool} during store maintenance failed: {e}");
252                return busy();
253            }
254        }
255    }
256}
257
258/// A remember, note or correct queued without a client id gets one here, so the
259/// caller can name it before it is stored. Text is redacted before it reaches the
260/// queue file, as the handler would before storing it.
261fn queued_entry(tool: &str, parts: &Parts, bytes: &Bytes) -> Result<Entry, String> {
262    let mut body: Option<Value> = if bytes.is_empty() {
263        None
264    } else {
265        Some(serde_json::from_slice(bytes).map_err(|e| format!("invalid JSON body: {e}"))?)
266    };
267    if names_its_write(tool) {
268        let Some(Value::Object(fields)) = body.as_mut() else {
269            return Err(format!("a {tool} needs a JSON object body"));
270        };
271        if !fields.get("id").is_some_and(Value::is_string) {
272            fields.insert(
273                "id".to_string(),
274                Value::String(uuid::Uuid::new_v4().to_string()),
275            );
276        }
277    }
278    if let Some(body) = body.as_mut() {
279        redact_strings(body);
280    }
281    Ok(Entry {
282        tool: tool.to_string(),
283        method: parts.method.to_string(),
284        path: parts.uri.path().to_string(),
285        query: parts.uri.query().map(redact_query),
286        body,
287    })
288}
289
290/// Writes whose reply carries the id of the row they create, so a queued
291/// one is given that id up front.
292fn names_its_write(tool: &str) -> bool {
293    matches!(tool, "remember" | "note" | "correct")
294}
295
296fn redact_strings(value: &mut Value) {
297    match value {
298        Value::String(s) => *s = redact(s).text,
299        Value::Array(items) => items.iter_mut().for_each(redact_strings),
300        Value::Object(fields) => fields.values_mut().for_each(redact_strings),
301        _ => {}
302    }
303}
304
305fn redact_query(query: &str) -> String {
306    let Ok(mut url) = reqwest::Url::parse(&format!("http://queue/?{query}")) else {
307        return redact(query).text;
308    };
309    let pairs: Vec<(String, String)> = url
310        .query_pairs()
311        .map(|(k, v)| (k.into_owned(), redact(&v).text))
312        .collect();
313    url.query_pairs_mut().clear().extend_pairs(pairs);
314    url.query().unwrap_or_default().to_string()
315}
316
317fn queued(entry: &Entry) -> Response {
318    let mut reply = json!({ "queued": true, "reason": QUEUED_REASON });
319    if names_its_write(&entry.tool)
320        && let Some(id) = entry.body.as_ref().and_then(|b| b["id"].as_str())
321    {
322        reply["id"] = Value::String(id.to_string());
323    }
324    (StatusCode::ACCEPTED, Json(reply)).into_response()
325}
326
327fn busy() -> Response {
328    let mut response = error(
329        StatusCode::SERVICE_UNAVAILABLE,
330        "store maintenance is running; retry later",
331    );
332    response
333        .headers_mut()
334        .insert(header::RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECS));
335    response
336}
337
338fn error(status: StatusCode, message: &str) -> Response {
339    (status, Json(json!({ "error": message }))).into_response()
340}
341
342#[cfg(test)]
343mod tests {
344    use super::*;
345
346    #[test]
347    fn writes_are_told_apart_from_reads() {
348        let post = Method::POST;
349        assert_eq!(classify(&Method::GET, "/v1/redaction"), Class::Read);
350        assert_eq!(classify(&post, "/v1/recall"), Class::Read);
351        assert_eq!(classify(&post, "/v1/maintenance/vacuum"), Class::Read);
352        assert_eq!(
353            classify(&post, "/v1/remember"),
354            Class::Queueable("remember")
355        );
356        assert_eq!(classify(&post, "/v1/notes"), Class::Queueable("note"));
357        assert_eq!(classify(&Method::GET, "/v1/notes"), Class::Read);
358        assert_eq!(
359            classify(&Method::DELETE, "/v1/memory/x"),
360            Class::Queueable("forget")
361        );
362        assert_eq!(classify(&post, "/v1/correct"), Class::Queueable("correct"));
363        assert_eq!(classify(&post, "/v1/sync/import"), Class::Refused);
364        assert_eq!(classify(&post, "/v1/some-new-route"), Class::Refused);
365    }
366
367    #[test]
368    fn a_queued_write_holds_no_secret() {
369        let token = ["gh", "p_"].concat() + "Qq7k3Jd9XwL2pVb8NzR4tYc6MhF1sGa5UeT0";
370        let remember = axum::http::Request::builder()
371            .method("POST")
372            .uri("/v1/remember")
373            .body(())
374            .unwrap()
375            .into_parts()
376            .0;
377        let body = json!({"content": format!("deploy with {token}"), "tags": [token.clone()]});
378        let entry = queued_entry("remember", &remember, &Bytes::from(body.to_string())).unwrap();
379        let forget = axum::http::Request::builder()
380            .method("DELETE")
381            .uri(format!("/v1/memory/x?hard=false&reason=leaked%20{token}"))
382            .body(())
383            .unwrap()
384            .into_parts()
385            .0;
386        let forget = queued_entry("forget", &forget, &Bytes::new()).unwrap();
387        let saved =
388            serde_json::to_string(&entry).unwrap() + &serde_json::to_string(&forget).unwrap();
389        assert!(!saved.contains(&token), "{saved}");
390        assert!(saved.contains("[REDACTED:github_token]"));
391        assert!(entry.body.unwrap()["id"].is_string());
392        assert!(forget.query.unwrap().starts_with("hard=false&reason="));
393    }
394
395    #[test]
396    fn a_queued_note_is_named_before_it_is_stored() {
397        let notes = axum::http::Request::builder()
398            .method("POST")
399            .uri("/v1/notes")
400            .body(())
401            .unwrap()
402            .into_parts()
403            .0;
404        let body = json!({"session_id": "s", "decision": "d"});
405        let entry = queued_entry("note", &notes, &Bytes::from(body.to_string())).unwrap();
406        let id = entry.body.as_ref().unwrap()["id"]
407            .as_str()
408            .unwrap()
409            .to_string();
410        assert_eq!(queued(&entry).status(), StatusCode::ACCEPTED);
411        let again = queued_entry(
412            "note",
413            &notes,
414            &Bytes::from(entry.body.unwrap().to_string()),
415        );
416        assert_eq!(again.unwrap().body.unwrap()["id"], id.as_str());
417    }
418
419    #[test]
420    fn a_queued_correct_gets_a_client_id_it_reports() {
421        let parts = axum::http::Request::builder()
422            .method("POST")
423            .uri("/v1/correct")
424            .body(())
425            .unwrap()
426            .into_parts()
427            .0;
428        let body = json!({"memory_id": "m", "content": "fixed", "evidence": "src/lib.rs:4"});
429        let entry = queued_entry("correct", &parts, &Bytes::from(body.to_string())).unwrap();
430        let id = entry.body.as_ref().unwrap()["id"]
431            .as_str()
432            .unwrap()
433            .to_string();
434        assert!(uuid::Uuid::parse_str(&id).is_ok());
435        let reply = queued(&entry);
436        assert_eq!(reply.status(), StatusCode::ACCEPTED);
437    }
438
439    #[test]
440    fn closing_waits_for_writes_in_flight() {
441        let gate = Arc::new(WriteGate::default());
442        let slot_gate = gate.clone();
443        let (entered, wait) = std::sync::mpsc::channel();
444        let writer = std::thread::spawn(move || {
445            let _slot = slot_gate.try_enter().unwrap();
446            entered.send(()).unwrap();
447            std::thread::sleep(Duration::from_millis(200));
448        });
449        wait.recv().unwrap();
450        let started = std::time::Instant::now();
451        let closed = gate.close();
452        assert!(started.elapsed() >= Duration::from_millis(100));
453        assert!(gate.try_enter().is_none());
454        writer.join().unwrap();
455        drop(closed);
456        assert!(gate.try_enter().is_some());
457    }
458
459    #[test]
460    fn after_maintenance_writes_queue_until_the_queue_drains() {
461        let gate = WriteGate::default();
462        drop(gate.close());
463        assert!(matches!(
464            gate.admit(Class::Queueable("remember"), false),
465            Admission::Queue
466        ));
467        assert!(matches!(
468            gate.admit(Class::Queueable("remember"), true),
469            Admission::Live(_)
470        ));
471        gate.open_when_drained(|| false);
472        assert!(matches!(
473            gate.admit(Class::Queueable("remember"), false),
474            Admission::Queue
475        ));
476        gate.open_when_drained(|| true);
477        assert!(matches!(
478            gate.admit(Class::Queueable("remember"), false),
479            Admission::Live(_)
480        ));
481    }
482}