Skip to main content

second_brain_api/
write_queue.rs

1//! Replays the writes the MCP server queued under `queue/` while this
2//! daemon was unreachable. Each entry is the request the MCP server would
3//! have sent; it goes through this daemon's own router, so auth, redaction
4//! and every handler apply exactly as for a live call.
5
6use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::time::{Duration, SystemTime};
13
14use axum::Router;
15use axum::body::Body;
16use axum::http::{Method, Request, StatusCode, header};
17use chrono::{DateTime, Utc};
18use fs4::fs_std::FileExt;
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use tower::ServiceExt;
22
23use crate::health::Health;
24use crate::write_gate::{REPLAY_HEADER, WriteGate};
25
26pub use second_brain_core::write_queue::{DIR_NAME, is_entry_name};
27pub const INTERVAL: Duration = Duration::from_secs(30);
28const LOCK_FILE: &str = ".lock";
29/// A writer that crashed between write and rename left it; a live writer
30/// renames within milliseconds.
31const STALE_TMP: Duration = Duration::from_secs(60 * 60);
32const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
33const ALIASES_FILE: &str = "aliases.json";
34const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
35/// The MCP server refuses to queue past the same count.
36const MAX_ENTRIES: usize = 10_000;
37
38/// One queued write, as the MCP server writes it.
39#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
40pub struct Entry {
41    pub tool: String,
42    pub method: String,
43    pub path: String,
44    /// Already URL-encoded, without the leading `?`.
45    #[serde(default)]
46    pub query: Option<String>,
47    #[serde(default)]
48    pub body: Option<Value>,
49}
50
51fn entry_time(name: &str) -> Option<DateTime<Utc>> {
52    let nanos: i64 = name.get(..19)?.parse().ok()?;
53    Some(DateTime::from_timestamp_nanos(nanos))
54}
55
56/// Entry file names, oldest first.
57pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
58    let mut names = Vec::new();
59    for item in std::fs::read_dir(dir)? {
60        let name = item?.file_name().to_string_lossy().into_owned();
61        if is_entry_name(&name) {
62            names.push(name);
63        }
64    }
65    names.sort();
66    Ok(names)
67}
68
69/// Saves a write for the replayer under the name the MCP server would give
70/// it, so both writers' entries replay in the order they were made.
71pub fn push(dir: &Path, entry: &Entry) -> anyhow::Result<()> {
72    std::fs::DirBuilder::new()
73        .recursive(true)
74        .mode(0o700)
75        .create(dir)?;
76    match entry_names(dir) {
77        Ok(names) if names.len() >= MAX_ENTRIES => {
78            anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
79        }
80        Ok(_) => {}
81        Err(e) => return Err(e.into()),
82    }
83    let nanos = SystemTime::now()
84        .duration_since(SystemTime::UNIX_EPOCH)?
85        .as_nanos();
86    let name = format!("{nanos:019}-{}.json", uuid::Uuid::new_v4());
87    let tmp = dir.join(format!("{name}.tmp"));
88    let mut file = std::fs::OpenOptions::new()
89        .write(true)
90        .create_new(true)
91        .mode(0o600)
92        .open(&tmp)?;
93    file.write_all(&serde_json::to_vec(entry)?)?;
94    file.sync_all()?;
95    std::fs::rename(&tmp, dir.join(name))?;
96    Ok(())
97}
98
99#[derive(Debug, Default, PartialEq)]
100pub struct Pass {
101    pub sent: usize,
102    pub rejected: usize,
103    /// The status or error that stopped the pass with entries left.
104    pub stopped: Option<String>,
105}
106
107pub struct WriteQueue {
108    dir: PathBuf,
109    app: Router,
110    token: String,
111    health: Arc<Health>,
112    gate: Arc<WriteGate>,
113}
114
115enum Verdict {
116    Done,
117    Rejected(StatusCode, String),
118    Retry(String),
119}
120
121impl WriteQueue {
122    pub fn new(
123        dir: PathBuf,
124        app: Router,
125        token: String,
126        health: Arc<Health>,
127        gate: Arc<WriteGate>,
128    ) -> Self {
129        Self {
130            dir,
131            app,
132            token,
133            health,
134            gate,
135        }
136    }
137
138    pub fn spawn(self) {
139        tokio::spawn(async move {
140            let mut interval = tokio::time::interval(INTERVAL);
141            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
142            loop {
143                tokio::select! {
144                    _ = interval.tick() => {}
145                    _ = self.gate.reopened() => {}
146                }
147                self.tick().await;
148            }
149        });
150    }
151
152    /// Replays what it can, unless store maintenance holds the writer or
153    /// another replayer on this `HOME` holds the lock, then reports the
154    /// queue as it stands to health.
155    pub async fn tick(&self) -> Option<Pass> {
156        if self.gate.is_closed() {
157            self.report();
158            return None;
159        }
160        let pass = match self.try_lock() {
161            Ok(Some(_lock)) => {
162                self.remove_stale_tmp();
163                Some(self.replay().await)
164            }
165            Ok(None) => None,
166            Err(e) => {
167                tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
168                None
169            }
170        };
171        self.gate.open_when_drained(|| self.is_empty());
172        self.report();
173        pass
174    }
175
176    fn is_empty(&self) -> bool {
177        match entry_names(&self.dir) {
178            Ok(names) => names.is_empty(),
179            Err(e) => e.kind() == std::io::ErrorKind::NotFound,
180        }
181    }
182
183    fn try_lock(&self) -> std::io::Result<Option<File>> {
184        if !self.dir.is_dir() {
185            return Ok(None);
186        }
187        let file = std::fs::OpenOptions::new()
188            .create(true)
189            .truncate(false)
190            .write(true)
191            .open(self.dir.join(LOCK_FILE))?;
192        Ok(file.try_lock_exclusive()?.then_some(file))
193    }
194
195    fn report(&self) {
196        let names = match entry_names(&self.dir) {
197            Ok(names) => names,
198            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
199            Err(e) => {
200                tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
201                return;
202            }
203        };
204        let oldest = names.first().and_then(|n| entry_time(n));
205        self.health
206            .queue_checked(names.len() as u64, oldest, Utc::now());
207    }
208
209    fn remove_stale_tmp(&self) {
210        let Ok(items) = std::fs::read_dir(&self.dir) else {
211            return;
212        };
213        let now = SystemTime::now();
214        for item in items.flatten() {
215            let path = item.path();
216            let stale = path.extension().is_some_and(|e| e == "tmp")
217                && item
218                    .metadata()
219                    .and_then(|m| m.modified())
220                    .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
221            if stale && let Err(e) = std::fs::remove_file(&path) {
222                tracing::warn!("cannot remove {}: {e}", path.display());
223            }
224        }
225    }
226
227    async fn replay(&self) -> Pass {
228        let mut pass = Pass::default();
229        let names = match entry_names(&self.dir) {
230            Ok(names) => names,
231            Err(e) => {
232                pass.stopped = Some(format!("listing the queue: {e}"));
233                return pass;
234            }
235        };
236        let mut aliases = Aliases::load(&self.dir);
237        for name in &names {
238            let path = self.dir.join(name);
239            let entry = match read_entry(&path) {
240                Ok(Some(entry)) => aliases.apply(entry),
241                Ok(None) => continue,
242                Err(e) => {
243                    set_aside(&path, &e);
244                    continue;
245                }
246            };
247            let (verdict, reply) = self.send(&entry).await;
248            match verdict {
249                Verdict::Done => {
250                    if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
251                        aliases.add(client, stored, Utc::now());
252                        if let Err(e) = aliases.save(&self.dir) {
253                            pass.stopped = Some(format!("saving the id aliases: {e:#}"));
254                            return pass;
255                        }
256                    }
257                    remove_entry(&path);
258                    pass.sent += 1;
259                }
260                Verdict::Rejected(status, message) => {
261                    tracing::warn!(
262                        "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
263                        entry.tool,
264                        status.as_u16()
265                    );
266                    self.health
267                        .write_rejected(&entry.tool, status.as_u16(), Utc::now());
268                    remove_entry(&path);
269                    pass.rejected += 1;
270                }
271                Verdict::Retry(why) => {
272                    tracing::info!("write queue replay paused at {name}: {why}");
273                    pass.stopped = Some(why);
274                    return pass;
275                }
276            }
277        }
278        if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
279            && aliases.prune(Utc::now())
280            && let Err(e) = aliases.save(&self.dir)
281        {
282            tracing::warn!("cannot prune the write queue id aliases: {e:#}");
283        }
284        pass
285    }
286
287    async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
288        let request = match build_request(entry, &self.token) {
289            Ok(r) => r,
290            Err(e) => {
291                return (
292                    Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
293                    None,
294                );
295            }
296        };
297        let response = match self.app.clone().oneshot(request).await {
298            Ok(r) => r,
299            Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
300        };
301        let status = response.status();
302        let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
303            Ok(b) => b,
304            Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
305        };
306        let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
307        let verdict =
308            if status.is_success() || (status == StatusCode::NOT_FOUND && is_hard_forget(entry)) {
309                Verdict::Done
310            } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
311                Verdict::Retry(format!("HTTP {status}"))
312            } else {
313                let message = reply
314                    .as_ref()
315                    .and_then(|v| v["error"].as_str().map(str::to_string))
316                    .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
317                Verdict::Rejected(status, message)
318            };
319        (verdict, reply)
320    }
321}
322
323fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
324    let method = Method::from_bytes(entry.method.as_bytes())?;
325    let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
326        Some(q) => format!("{}?{q}", entry.path),
327        None => entry.path.clone(),
328    };
329    let builder = Request::builder()
330        .method(method)
331        .uri(uri)
332        .header(header::AUTHORIZATION, format!("Bearer {token}"))
333        .header(REPLAY_HEADER, "1");
334    Ok(match &entry.body {
335        Some(body) => builder
336            .header(header::CONTENT_TYPE, "application/json")
337            .body(Body::from(body.to_string()))?,
338        None => builder.body(Body::empty())?,
339    })
340}
341
342fn is_hard_forget(entry: &Entry) -> bool {
343    entry.method.eq_ignore_ascii_case("DELETE")
344        && entry
345            .query
346            .as_deref()
347            .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
348}
349
350/// `(client id, stored id)` when a replayed remember landed as another row
351/// because its content was already stored.
352fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
353    if entry.tool != "remember" {
354        return None;
355    }
356    let sent = entry.body.as_ref()?["id"].as_str()?;
357    let stored = reply?["id"].as_str()?;
358    (sent != stored).then(|| (sent.to_string(), stored.to_string()))
359}
360
361/// Client ids of replayed remembers that landed as an existing row, mapped
362/// to that row. Applied to each entry as it is sent rather than rewritten
363/// into the files, so an entry queued while the remember was in flight
364/// (after this pass listed the queue) still reaches the stored row.
365#[derive(Default, Serialize, Deserialize)]
366struct Aliases(BTreeMap<String, Alias>);
367
368#[derive(Serialize, Deserialize)]
369struct Alias {
370    to: String,
371    at: DateTime<Utc>,
372}
373
374impl Aliases {
375    fn load(dir: &Path) -> Self {
376        match std::fs::read(dir.join(ALIASES_FILE)) {
377            Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
378                tracing::warn!("ignoring unreadable write queue id aliases: {e}");
379                Self::default()
380            }),
381            Err(_) => Self::default(),
382        }
383    }
384
385    fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
386        self.0.insert(
387            client,
388            Alias {
389                to: stored,
390                at: now,
391            },
392        );
393    }
394
395    fn apply(&self, mut entry: Entry) -> Entry {
396        for (client, alias) in &self.0 {
397            entry.path = entry.path.replace(client.as_str(), &alias.to);
398            entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
399            if let Some(body) = entry.body.as_mut() {
400                replace_strings(body, client, &alias.to);
401            }
402        }
403        entry
404    }
405
406    /// With the queue empty, a write naming a client id is sent live, where
407    /// a 404 for a note that landed as an older row is accepted; an hour
408    /// covers a session that saw the queue non-empty just before it emptied.
409    fn prune(&mut self, now: DateTime<Utc>) -> bool {
410        let before = self.0.len();
411        self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
412        self.0.len() != before
413    }
414
415    fn save(&self, dir: &Path) -> anyhow::Result<()> {
416        let path = dir.join(ALIASES_FILE);
417        if self.0.is_empty() {
418            return match std::fs::remove_file(&path) {
419                Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
420                _ => Ok(()),
421            };
422        }
423        let tmp = dir.join(format!(
424            "{ALIASES_FILE}.{}.tmp",
425            uuid::Uuid::new_v4().simple()
426        ));
427        let mut file = std::fs::OpenOptions::new()
428            .write(true)
429            .create_new(true)
430            .mode(0o600)
431            .open(&tmp)?;
432        file.write_all(&serde_json::to_vec(self)?)?;
433        file.sync_all()?;
434        std::fs::rename(&tmp, path)?;
435        Ok(())
436    }
437}
438
439/// `Ok(None)` when another replayer or the MCP server's ingest collapse
440/// removed it after the listing.
441fn read_entry(path: &Path) -> anyhow::Result<Option<Entry>> {
442    match std::fs::read(path) {
443        Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
444        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
445        Err(e) => Err(e.into()),
446    }
447}
448
449/// Renamed out of the entry namespace rather than deleted, so the write it
450/// held can still be read by hand.
451fn set_aside(path: &Path, error: &anyhow::Error) {
452    let aside = path.with_extension("unreadable");
453    tracing::warn!(
454        "write queue entry {} is unreadable ({error:#}); moved to {}",
455        path.display(),
456        aside.display()
457    );
458    if let Err(e) = std::fs::rename(path, &aside) {
459        tracing::warn!("cannot move {} aside: {e}", path.display());
460    }
461}
462
463fn remove_entry(path: &Path) {
464    if let Err(e) = std::fs::remove_file(path)
465        && e.kind() != std::io::ErrorKind::NotFound
466    {
467        tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
468    }
469}
470
471fn replace_strings(value: &mut Value, from: &str, to: &str) {
472    match value {
473        Value::String(s) if s.contains(from) => *s = s.replace(from, to),
474        Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
475        Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
476        _ => {}
477    }
478}
479
480#[cfg(test)]
481mod tests {
482    use super::*;
483
484    #[test]
485    fn an_entry_name_dates_the_call() {
486        let at = entry_time("1793505622000000000-x.json").unwrap();
487        assert_eq!(at.timestamp(), 1_793_505_622);
488    }
489
490    #[test]
491    fn only_a_hard_forget_treats_404_as_done() {
492        let forget = |query: &str| Entry {
493            tool: "forget".to_string(),
494            method: "DELETE".to_string(),
495            path: "/v1/memory/x".to_string(),
496            query: Some(query.to_string()),
497            body: None,
498        };
499        assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
500        assert!(!is_hard_forget(&forget("hard=false")));
501    }
502}