Skip to main content

second_brain_api/
write_queue.rs

1//! Replays the writes the MCP server queued under `queue/` while this
2//! daemon was unreachable. Each entry is the request the MCP server would
3//! have sent; it goes through this daemon's own router, so auth, redaction
4//! and every handler apply exactly as for a live call.
5
6use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::time::{Duration, SystemTime};
13
14use axum::Router;
15use axum::body::Body;
16use axum::http::{Method, Request, StatusCode, header};
17use chrono::{DateTime, Utc};
18use fs4::fs_std::FileExt;
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use tower::ServiceExt;
22
23use crate::health::Health;
24use crate::write_gate::{REPLAY_HEADER, WriteGate};
25
26pub const DIR_NAME: &str = "queue";
27pub const INTERVAL: Duration = Duration::from_secs(30);
28const LOCK_FILE: &str = ".lock";
29/// A writer that crashed between write and rename left it; a live writer
30/// renames within milliseconds.
31const STALE_TMP: Duration = Duration::from_secs(60 * 60);
32const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
33const ALIASES_FILE: &str = "aliases.json";
34const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
35/// The MCP server refuses to queue past the same count.
36const MAX_ENTRIES: usize = 10_000;
37
38/// One queued write, as the MCP server writes it.
39#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
40pub struct Entry {
41    pub tool: String,
42    pub method: String,
43    pub path: String,
44    /// Already URL-encoded, without the leading `?`.
45    #[serde(default)]
46    pub query: Option<String>,
47    #[serde(default)]
48    pub body: Option<Value>,
49}
50
51/// `<19-digit unix nanos>-<uuid>.json`; no other name in the directory is
52/// an entry.
53pub fn is_entry_name(name: &str) -> bool {
54    let Some(stem) = name.strip_suffix(".json") else {
55        return false;
56    };
57    let Some((nanos, id)) = stem.split_at_checked(19) else {
58        return false;
59    };
60    nanos.bytes().all(|b| b.is_ascii_digit())
61        && id
62            .strip_prefix('-')
63            .is_some_and(|id| id.len() == 36 && uuid::Uuid::parse_str(id).is_ok())
64}
65
66fn entry_time(name: &str) -> Option<DateTime<Utc>> {
67    let nanos: i64 = name.get(..19)?.parse().ok()?;
68    Some(DateTime::from_timestamp_nanos(nanos))
69}
70
71/// Entry file names, oldest first.
72pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
73    let mut names = Vec::new();
74    for item in std::fs::read_dir(dir)? {
75        let name = item?.file_name().to_string_lossy().into_owned();
76        if is_entry_name(&name) {
77            names.push(name);
78        }
79    }
80    names.sort();
81    Ok(names)
82}
83
84/// Saves a write for the replayer under the name the MCP server would give
85/// it, so both writers' entries replay in the order they were made.
86pub fn push(dir: &Path, entry: &Entry) -> anyhow::Result<()> {
87    std::fs::DirBuilder::new()
88        .recursive(true)
89        .mode(0o700)
90        .create(dir)?;
91    match entry_names(dir) {
92        Ok(names) if names.len() >= MAX_ENTRIES => {
93            anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
94        }
95        Ok(_) => {}
96        Err(e) => return Err(e.into()),
97    }
98    let nanos = SystemTime::now()
99        .duration_since(SystemTime::UNIX_EPOCH)?
100        .as_nanos();
101    let name = format!("{nanos:019}-{}.json", uuid::Uuid::new_v4());
102    let tmp = dir.join(format!("{name}.tmp"));
103    let mut file = std::fs::OpenOptions::new()
104        .write(true)
105        .create_new(true)
106        .mode(0o600)
107        .open(&tmp)?;
108    file.write_all(&serde_json::to_vec(entry)?)?;
109    file.sync_all()?;
110    std::fs::rename(&tmp, dir.join(name))?;
111    Ok(())
112}
113
114#[derive(Debug, Default, PartialEq)]
115pub struct Pass {
116    pub sent: usize,
117    pub rejected: usize,
118    /// The status or error that stopped the pass with entries left.
119    pub stopped: Option<String>,
120}
121
122pub struct WriteQueue {
123    dir: PathBuf,
124    app: Router,
125    token: String,
126    health: Arc<Health>,
127    gate: Arc<WriteGate>,
128}
129
130enum Verdict {
131    Done,
132    Rejected(StatusCode, String),
133    Retry(String),
134}
135
136impl WriteQueue {
137    pub fn new(
138        dir: PathBuf,
139        app: Router,
140        token: String,
141        health: Arc<Health>,
142        gate: Arc<WriteGate>,
143    ) -> Self {
144        Self {
145            dir,
146            app,
147            token,
148            health,
149            gate,
150        }
151    }
152
153    pub fn spawn(self) {
154        tokio::spawn(async move {
155            let mut interval = tokio::time::interval(INTERVAL);
156            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
157            loop {
158                tokio::select! {
159                    _ = interval.tick() => {}
160                    _ = self.gate.reopened() => {}
161                }
162                self.tick().await;
163            }
164        });
165    }
166
167    /// Replays what it can, unless store maintenance holds the writer or
168    /// another replayer on this `HOME` holds the lock, then reports the
169    /// queue as it stands to health.
170    pub async fn tick(&self) -> Option<Pass> {
171        if self.gate.is_closed() {
172            self.report();
173            return None;
174        }
175        let pass = match self.try_lock() {
176            Ok(Some(_lock)) => {
177                self.remove_stale_tmp();
178                Some(self.replay().await)
179            }
180            Ok(None) => None,
181            Err(e) => {
182                tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
183                None
184            }
185        };
186        self.gate.open_when_drained(|| self.is_empty());
187        self.report();
188        pass
189    }
190
191    fn is_empty(&self) -> bool {
192        match entry_names(&self.dir) {
193            Ok(names) => names.is_empty(),
194            Err(e) => e.kind() == std::io::ErrorKind::NotFound,
195        }
196    }
197
198    fn try_lock(&self) -> std::io::Result<Option<File>> {
199        if !self.dir.is_dir() {
200            return Ok(None);
201        }
202        let file = std::fs::OpenOptions::new()
203            .create(true)
204            .truncate(false)
205            .write(true)
206            .open(self.dir.join(LOCK_FILE))?;
207        Ok(file.try_lock_exclusive()?.then_some(file))
208    }
209
210    fn report(&self) {
211        let names = match entry_names(&self.dir) {
212            Ok(names) => names,
213            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
214            Err(e) => {
215                tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
216                return;
217            }
218        };
219        let oldest = names.first().and_then(|n| entry_time(n));
220        self.health
221            .queue_checked(names.len() as u64, oldest, Utc::now());
222    }
223
224    fn remove_stale_tmp(&self) {
225        let Ok(items) = std::fs::read_dir(&self.dir) else {
226            return;
227        };
228        let now = SystemTime::now();
229        for item in items.flatten() {
230            let path = item.path();
231            let stale = path.extension().is_some_and(|e| e == "tmp")
232                && item
233                    .metadata()
234                    .and_then(|m| m.modified())
235                    .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
236            if stale && let Err(e) = std::fs::remove_file(&path) {
237                tracing::warn!("cannot remove {}: {e}", path.display());
238            }
239        }
240    }
241
242    async fn replay(&self) -> Pass {
243        let mut pass = Pass::default();
244        let names = match entry_names(&self.dir) {
245            Ok(names) => names,
246            Err(e) => {
247                pass.stopped = Some(format!("listing the queue: {e}"));
248                return pass;
249            }
250        };
251        let mut aliases = Aliases::load(&self.dir);
252        for name in &names {
253            let path = self.dir.join(name);
254            let entry = match read_entry(&path) {
255                Ok(Some(entry)) => aliases.apply(entry),
256                Ok(None) => continue,
257                Err(e) => {
258                    set_aside(&path, &e);
259                    continue;
260                }
261            };
262            let (verdict, reply) = self.send(&entry).await;
263            match verdict {
264                Verdict::Done => {
265                    if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
266                        aliases.add(client, stored, Utc::now());
267                        if let Err(e) = aliases.save(&self.dir) {
268                            pass.stopped = Some(format!("saving the id aliases: {e:#}"));
269                            return pass;
270                        }
271                    }
272                    remove_entry(&path);
273                    pass.sent += 1;
274                }
275                Verdict::Rejected(status, message) => {
276                    tracing::warn!(
277                        "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
278                        entry.tool,
279                        status.as_u16()
280                    );
281                    self.health
282                        .write_rejected(&entry.tool, status.as_u16(), Utc::now());
283                    remove_entry(&path);
284                    pass.rejected += 1;
285                }
286                Verdict::Retry(why) => {
287                    tracing::info!("write queue replay paused at {name}: {why}");
288                    pass.stopped = Some(why);
289                    return pass;
290                }
291            }
292        }
293        if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
294            && aliases.prune(Utc::now())
295            && let Err(e) = aliases.save(&self.dir)
296        {
297            tracing::warn!("cannot prune the write queue id aliases: {e:#}");
298        }
299        pass
300    }
301
302    async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
303        let request = match build_request(entry, &self.token) {
304            Ok(r) => r,
305            Err(e) => {
306                return (
307                    Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
308                    None,
309                );
310            }
311        };
312        let response = match self.app.clone().oneshot(request).await {
313            Ok(r) => r,
314            Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
315        };
316        let status = response.status();
317        let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
318            Ok(b) => b,
319            Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
320        };
321        let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
322        let verdict =
323            if status.is_success() || (status == StatusCode::NOT_FOUND && is_hard_forget(entry)) {
324                Verdict::Done
325            } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
326                Verdict::Retry(format!("HTTP {status}"))
327            } else {
328                let message = reply
329                    .as_ref()
330                    .and_then(|v| v["error"].as_str().map(str::to_string))
331                    .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
332                Verdict::Rejected(status, message)
333            };
334        (verdict, reply)
335    }
336}
337
338fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
339    let method = Method::from_bytes(entry.method.as_bytes())?;
340    let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
341        Some(q) => format!("{}?{q}", entry.path),
342        None => entry.path.clone(),
343    };
344    let builder = Request::builder()
345        .method(method)
346        .uri(uri)
347        .header(header::AUTHORIZATION, format!("Bearer {token}"))
348        .header(REPLAY_HEADER, "1");
349    Ok(match &entry.body {
350        Some(body) => builder
351            .header(header::CONTENT_TYPE, "application/json")
352            .body(Body::from(body.to_string()))?,
353        None => builder.body(Body::empty())?,
354    })
355}
356
357fn is_hard_forget(entry: &Entry) -> bool {
358    entry.method.eq_ignore_ascii_case("DELETE")
359        && entry
360            .query
361            .as_deref()
362            .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
363}
364
365/// `(client id, stored id)` when a replayed remember landed as another row
366/// because its content was already stored.
367fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
368    if entry.tool != "remember" {
369        return None;
370    }
371    let sent = entry.body.as_ref()?["id"].as_str()?;
372    let stored = reply?["id"].as_str()?;
373    (sent != stored).then(|| (sent.to_string(), stored.to_string()))
374}
375
376/// Client ids of replayed remembers that landed as an existing row, mapped
377/// to that row. Applied to each entry as it is sent rather than rewritten
378/// into the files, so an entry queued while the remember was in flight
379/// (after this pass listed the queue) still reaches the stored row.
380#[derive(Default, Serialize, Deserialize)]
381struct Aliases(BTreeMap<String, Alias>);
382
383#[derive(Serialize, Deserialize)]
384struct Alias {
385    to: String,
386    at: DateTime<Utc>,
387}
388
389impl Aliases {
390    fn load(dir: &Path) -> Self {
391        match std::fs::read(dir.join(ALIASES_FILE)) {
392            Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
393                tracing::warn!("ignoring unreadable write queue id aliases: {e}");
394                Self::default()
395            }),
396            Err(_) => Self::default(),
397        }
398    }
399
400    fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
401        self.0.insert(
402            client,
403            Alias {
404                to: stored,
405                at: now,
406            },
407        );
408    }
409
410    fn apply(&self, mut entry: Entry) -> Entry {
411        for (client, alias) in &self.0 {
412            entry.path = entry.path.replace(client.as_str(), &alias.to);
413            entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
414            if let Some(body) = entry.body.as_mut() {
415                replace_strings(body, client, &alias.to);
416            }
417        }
418        entry
419    }
420
421    /// With the queue empty, a write naming a client id is sent live, where
422    /// a 404 for a note that landed as an older row is accepted; an hour
423    /// covers a session that saw the queue non-empty just before it emptied.
424    fn prune(&mut self, now: DateTime<Utc>) -> bool {
425        let before = self.0.len();
426        self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
427        self.0.len() != before
428    }
429
430    fn save(&self, dir: &Path) -> anyhow::Result<()> {
431        let path = dir.join(ALIASES_FILE);
432        if self.0.is_empty() {
433            return match std::fs::remove_file(&path) {
434                Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
435                _ => Ok(()),
436            };
437        }
438        let tmp = dir.join(format!(
439            "{ALIASES_FILE}.{}.tmp",
440            uuid::Uuid::new_v4().simple()
441        ));
442        let mut file = std::fs::OpenOptions::new()
443            .write(true)
444            .create_new(true)
445            .mode(0o600)
446            .open(&tmp)?;
447        file.write_all(&serde_json::to_vec(self)?)?;
448        file.sync_all()?;
449        std::fs::rename(&tmp, path)?;
450        Ok(())
451    }
452}
453
454/// `Ok(None)` when another replayer or the MCP server's ingest collapse
455/// removed it after the listing.
456fn read_entry(path: &Path) -> anyhow::Result<Option<Entry>> {
457    match std::fs::read(path) {
458        Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
459        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
460        Err(e) => Err(e.into()),
461    }
462}
463
464/// Renamed out of the entry namespace rather than deleted, so the write it
465/// held can still be read by hand.
466fn set_aside(path: &Path, error: &anyhow::Error) {
467    let aside = path.with_extension("unreadable");
468    tracing::warn!(
469        "write queue entry {} is unreadable ({error:#}); moved to {}",
470        path.display(),
471        aside.display()
472    );
473    if let Err(e) = std::fs::rename(path, &aside) {
474        tracing::warn!("cannot move {} aside: {e}", path.display());
475    }
476}
477
478fn remove_entry(path: &Path) {
479    if let Err(e) = std::fs::remove_file(path)
480        && e.kind() != std::io::ErrorKind::NotFound
481    {
482        tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
483    }
484}
485
486fn replace_strings(value: &mut Value, from: &str, to: &str) {
487    match value {
488        Value::String(s) if s.contains(from) => *s = s.replace(from, to),
489        Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
490        Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
491        _ => {}
492    }
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498
499    #[test]
500    fn only_timestamped_uuid_json_names_are_entries() {
501        let id = "0b5c2f9e-5d0e-4c1a-9a57-0c7f4f1f2a10";
502        assert!(is_entry_name(&format!("1793505622000000000-{id}.json")));
503        for name in [
504            "projects.json".to_string(),
505            ".lock".to_string(),
506            format!("1793505622000000000-{id}.json.tmp"),
507            format!("179350562200000000-{id}.json"),
508            format!("1793505622000000000-{id}.unreadable"),
509            "1793505622000000000-not-a-uuid.json".to_string(),
510        ] {
511            assert!(!is_entry_name(&name), "{name}");
512        }
513    }
514
515    #[test]
516    fn an_entry_name_dates_the_call() {
517        let at = entry_time("1793505622000000000-x.json").unwrap();
518        assert_eq!(at.timestamp(), 1_793_505_622);
519    }
520
521    #[test]
522    fn only_a_hard_forget_treats_404_as_done() {
523        let forget = |query: &str| Entry {
524            tool: "forget".to_string(),
525            method: "DELETE".to_string(),
526            path: "/v1/memory/x".to_string(),
527            query: Some(query.to_string()),
528            body: None,
529        };
530        assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
531        assert!(!is_hard_forget(&forget("hard=false")));
532    }
533}