Skip to main content

second_brain_api/
write_queue.rs

1//! Replays the writes the MCP server queued under `queue/` while this
2//! daemon was unreachable. Each entry is the request the MCP server would
3//! have sent; it goes through this daemon's own router, so auth, redaction
4//! and every handler apply exactly as for a live call.
5
6use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::{Duration, SystemTime};
14
15use age::x25519::{Identity, Recipient};
16use axum::Router;
17use axum::body::Body;
18use axum::http::{Method, Request, StatusCode, header};
19use chrono::{DateTime, Utc};
20use fs4::fs_std::FileExt;
21use second_brain_core::keys::{self, KeyStorage};
22use second_brain_core::write_queue::{OpenError, entry_name, is_sealed_name, open};
23use serde::{Deserialize, Serialize};
24use serde_json::Value;
25use tower::ServiceExt;
26
27use crate::health::Health;
28use crate::write_gate::{REPLAY_HEADER, WriteGate};
29
30pub use second_brain_core::write_queue::{DIR_NAME, Entry, is_entry_name};
31pub const INTERVAL: Duration = Duration::from_secs(30);
32const LOCK_FILE: &str = ".lock";
33/// A writer that crashed between write and rename left it; a live writer
34/// renames within milliseconds.
35const STALE_TMP: Duration = Duration::from_secs(60 * 60);
36const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
37const ALIASES_FILE: &str = "aliases.json";
38const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
39/// The MCP server refuses to queue past the same count.
40const MAX_ENTRIES: usize = 10_000;
41
42fn entry_time(name: &str) -> Option<DateTime<Utc>> {
43    let nanos: i64 = name.get(..19)?.parse().ok()?;
44    Some(DateTime::from_timestamp_nanos(nanos))
45}
46
47/// Entry file names, oldest first.
48pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
49    let mut names = Vec::new();
50    for item in std::fs::read_dir(dir)? {
51        let name = item?.file_name().to_string_lossy().into_owned();
52        if is_entry_name(&name) {
53            names.push(name);
54        }
55    }
56    names.sort();
57    Ok(names)
58}
59
60/// This device's age identity, which seals and opens queued writes, after
61/// publishing its recipient for the MCP server; None, logged, when the device
62/// keys can't be read, and writes then queue in plaintext.
63pub fn publish_device_identity(data_dir: &Path, storage: &KeyStorage) -> Option<Identity> {
64    let keys = match keys::load_device_keys_in(data_dir, storage) {
65        Ok(Some(keys)) => keys,
66        Ok(None) => {
67            tracing::warn!("no device key: queued writes are not sealed");
68            return None;
69        }
70        Err(e) => {
71            tracing::error!("device keys unreadable: queued writes are not sealed: {e:#}");
72            return None;
73        }
74    };
75    if let Err(e) = keys::publish_device_pub(data_dir, &keys) {
76        tracing::warn!("the MCP server cannot seal the writes it queues: {e:#}");
77    }
78    Some(keys.age_identity().clone())
79}
80
81/// Saves a write for the replayer under the name the MCP server would give
82/// it, so both writers' entries replay in the order they were made; sealed
83/// when there is a recipient.
84pub fn push(dir: &Path, entry: &Entry, recipient: Option<&Recipient>) -> anyhow::Result<()> {
85    std::fs::DirBuilder::new()
86        .recursive(true)
87        .mode(0o700)
88        .create(dir)?;
89    match entry_names(dir) {
90        Ok(names) if names.len() >= MAX_ENTRIES => {
91            anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
92        }
93        Ok(_) => {}
94        Err(e) => return Err(e.into()),
95    }
96    let nanos = SystemTime::now()
97        .duration_since(SystemTime::UNIX_EPOCH)?
98        .as_nanos();
99    let bytes = match recipient {
100        Some(recipient) => entry.to_sealed_file(std::slice::from_ref(recipient))?,
101        None => entry.to_plain_file()?,
102    };
103    let name = entry_name(nanos, recipient.is_some());
104    let tmp = dir.join(format!("{name}.tmp"));
105    let mut file = std::fs::OpenOptions::new()
106        .write(true)
107        .create_new(true)
108        .mode(0o600)
109        .open(&tmp)?;
110    file.write_all(&bytes)?;
111    file.sync_all()?;
112    std::fs::rename(&tmp, dir.join(name))?;
113    Ok(())
114}
115
116#[derive(Debug, Default, PartialEq)]
117pub struct Pass {
118    pub sent: usize,
119    pub rejected: usize,
120    /// Sealed entries left in place because this daemon has no device key.
121    pub waiting: usize,
122    /// The status or error that stopped the pass with entries left.
123    pub stopped: Option<String>,
124}
125
126pub struct WriteQueue {
127    dir: PathBuf,
128    app: Router,
129    token: String,
130    health: Arc<Health>,
131    gate: Arc<WriteGate>,
132    identity: Option<Identity>,
133    warned_no_key: AtomicBool,
134}
135
136enum Verdict {
137    Done,
138    Rejected(StatusCode, String),
139    Retry(String),
140}
141
142impl WriteQueue {
143    pub fn new(
144        dir: PathBuf,
145        app: Router,
146        token: String,
147        health: Arc<Health>,
148        gate: Arc<WriteGate>,
149    ) -> Self {
150        Self {
151            dir,
152            app,
153            token,
154            health,
155            gate,
156            identity: None,
157            warned_no_key: AtomicBool::new(false),
158        }
159    }
160
161    /// Opens entries sealed to this device; without it sealed entries stay
162    /// in place and the rest replay past them.
163    pub fn opening_with(mut self, identity: Identity) -> Self {
164        self.identity = Some(identity);
165        self
166    }
167
168    pub fn spawn(self) {
169        tokio::spawn(async move {
170            let mut interval = tokio::time::interval(INTERVAL);
171            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
172            loop {
173                tokio::select! {
174                    _ = interval.tick() => {}
175                    _ = self.gate.reopened() => {}
176                }
177                self.tick().await;
178            }
179        });
180    }
181
182    /// Replays what it can, unless store maintenance holds the writer or
183    /// another replayer on this `HOME` holds the lock, then reports the
184    /// queue as it stands to health.
185    pub async fn tick(&self) -> Option<Pass> {
186        if self.gate.is_closed() {
187            self.report();
188            return None;
189        }
190        let pass = match self.try_lock() {
191            Ok(Some(_lock)) => {
192                self.remove_stale_tmp();
193                Some(self.replay().await)
194            }
195            Ok(None) => None,
196            Err(e) => {
197                tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
198                None
199            }
200        };
201        self.gate.open_when_drained(|| self.nothing_to_replay());
202        self.report();
203        pass
204    }
205
206    /// Sealed entries waiting for a device key don't count, so they never
207    /// hold the gate.
208    fn nothing_to_replay(&self) -> bool {
209        match entry_names(&self.dir) {
210            Ok(names) => names.iter().all(|name| self.waits_for_key(name)),
211            Err(e) => e.kind() == std::io::ErrorKind::NotFound,
212        }
213    }
214
215    fn waits_for_key(&self, name: &str) -> bool {
216        self.identity.is_none() && is_sealed_name(name)
217    }
218
219    fn try_lock(&self) -> std::io::Result<Option<File>> {
220        if !self.dir.is_dir() {
221            return Ok(None);
222        }
223        let file = std::fs::OpenOptions::new()
224            .create(true)
225            .truncate(false)
226            .write(true)
227            .open(self.dir.join(LOCK_FILE))?;
228        Ok(file.try_lock_exclusive()?.then_some(file))
229    }
230
231    fn report(&self) {
232        let names = match entry_names(&self.dir) {
233            Ok(names) => names,
234            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
235            Err(e) => {
236                tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
237                return;
238            }
239        };
240        let oldest = names.first().and_then(|n| entry_time(n));
241        self.health
242            .queue_checked(names.len() as u64, oldest, Utc::now());
243    }
244
245    fn remove_stale_tmp(&self) {
246        let Ok(items) = std::fs::read_dir(&self.dir) else {
247            return;
248        };
249        let now = SystemTime::now();
250        for item in items.flatten() {
251            let path = item.path();
252            let stale = path.extension().is_some_and(|e| e == "tmp")
253                && item
254                    .metadata()
255                    .and_then(|m| m.modified())
256                    .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
257            if stale && let Err(e) = std::fs::remove_file(&path) {
258                tracing::warn!("cannot remove {}: {e}", path.display());
259            }
260        }
261    }
262
263    async fn replay(&self) -> Pass {
264        let mut pass = Pass::default();
265        let names = match entry_names(&self.dir) {
266            Ok(names) => names,
267            Err(e) => {
268                pass.stopped = Some(format!("listing the queue: {e}"));
269                return pass;
270            }
271        };
272        let mut aliases = Aliases::load(&self.dir);
273        for name in &names {
274            let path = self.dir.join(name);
275            let entry = match read_entry(&path, self.identity.as_ref()) {
276                Ok(Some(entry)) => aliases.apply(entry),
277                Ok(None) => continue,
278                Err(OpenError::NoKey) => {
279                    pass.waiting += 1;
280                    continue;
281                }
282                Err(OpenError::Unreadable(e)) => {
283                    set_aside(&path, &e);
284                    continue;
285                }
286            };
287            let (verdict, reply) = self.send(&entry).await;
288            match verdict {
289                Verdict::Done => {
290                    if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
291                        aliases.add(client, stored, Utc::now());
292                        if let Err(e) = aliases.save(&self.dir) {
293                            pass.stopped = Some(format!("saving the id aliases: {e:#}"));
294                            return pass;
295                        }
296                    }
297                    remove_entry(&path);
298                    pass.sent += 1;
299                }
300                Verdict::Rejected(status, message) => {
301                    tracing::warn!(
302                        "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
303                        entry.tool,
304                        status.as_u16()
305                    );
306                    self.health
307                        .write_rejected(&entry.tool, status.as_u16(), Utc::now());
308                    remove_entry(&path);
309                    pass.rejected += 1;
310                }
311                Verdict::Retry(why) => {
312                    tracing::info!("write queue replay paused at {name}: {why}");
313                    pass.stopped = Some(why);
314                    return pass;
315                }
316            }
317        }
318        if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
319            && aliases.prune(Utc::now())
320            && let Err(e) = aliases.save(&self.dir)
321        {
322            tracing::warn!("cannot prune the write queue id aliases: {e:#}");
323        }
324        if pass.waiting > 0 && !self.warned_no_key.swap(true, Ordering::Relaxed) {
325            tracing::warn!(
326                "{} sealed queued writes wait in {} for the device key, which this daemon \
327                 could not read at start; later writes replay without them",
328                pass.waiting,
329                self.dir.display()
330            );
331        }
332        pass
333    }
334
335    async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
336        let request = match build_request(entry, &self.token) {
337            Ok(r) => r,
338            Err(e) => {
339                return (
340                    Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
341                    None,
342                );
343            }
344        };
345        let response = match self.app.clone().oneshot(request).await {
346            Ok(r) => r,
347            Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
348        };
349        let status = response.status();
350        let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
351            Ok(b) => b,
352            Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
353        };
354        let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
355        let verdict = if status.is_success()
356            || (status == StatusCode::NOT_FOUND && is_idempotent_removal(entry))
357        {
358            Verdict::Done
359        } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
360            Verdict::Retry(format!("HTTP {status}"))
361        } else {
362            let message = reply
363                .as_ref()
364                .and_then(|v| v["error"].as_str().map(str::to_string))
365                .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
366            Verdict::Rejected(status, message)
367        };
368        (verdict, reply)
369    }
370}
371
372fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
373    let method = Method::from_bytes(entry.method.as_bytes())?;
374    let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
375        Some(q) => format!("{}?{q}", entry.path),
376        None => entry.path.clone(),
377    };
378    let builder = Request::builder()
379        .method(method)
380        .uri(uri)
381        .header(header::AUTHORIZATION, format!("Bearer {token}"))
382        .header(REPLAY_HEADER, "1");
383    Ok(match &entry.body {
384        Some(body) => builder
385            .header(header::CONTENT_TYPE, "application/json")
386            .body(Body::from(body.to_string()))?,
387        None => builder.body(Body::empty())?,
388    })
389}
390
391fn is_idempotent_removal(entry: &Entry) -> bool {
392    is_hard_forget(entry) || is_delete(entry)
393}
394
395fn is_delete(entry: &Entry) -> bool {
396    entry.method.eq_ignore_ascii_case("POST") && entry.path == "/v1/delete"
397}
398
399fn is_hard_forget(entry: &Entry) -> bool {
400    entry.method.eq_ignore_ascii_case("DELETE")
401        && entry
402            .query
403            .as_deref()
404            .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
405}
406
407/// `(client id, stored id)` when a replayed remember landed as another row
408/// because its content was already stored.
409fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
410    if entry.tool != "remember" {
411        return None;
412    }
413    let sent = entry.body.as_ref()?["id"].as_str()?;
414    let stored = reply?["id"].as_str()?;
415    (sent != stored).then(|| (sent.to_string(), stored.to_string()))
416}
417
418/// Client ids of replayed remembers that landed as an existing row, mapped
419/// to that row. Applied to each entry as it is sent rather than rewritten
420/// into the files, so an entry queued while the remember was in flight
421/// (after this pass listed the queue) still reaches the stored row.
422#[derive(Default, Serialize, Deserialize)]
423struct Aliases(BTreeMap<String, Alias>);
424
425#[derive(Serialize, Deserialize)]
426struct Alias {
427    to: String,
428    at: DateTime<Utc>,
429}
430
431impl Aliases {
432    fn load(dir: &Path) -> Self {
433        match std::fs::read(dir.join(ALIASES_FILE)) {
434            Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
435                tracing::warn!("ignoring unreadable write queue id aliases: {e}");
436                Self::default()
437            }),
438            Err(_) => Self::default(),
439        }
440    }
441
442    fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
443        self.0.insert(
444            client,
445            Alias {
446                to: stored,
447                at: now,
448            },
449        );
450    }
451
452    fn apply(&self, mut entry: Entry) -> Entry {
453        for (client, alias) in &self.0 {
454            entry.path = entry.path.replace(client.as_str(), &alias.to);
455            entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
456            if let Some(body) = entry.body.as_mut() {
457                replace_strings(body, client, &alias.to);
458            }
459        }
460        entry
461    }
462
463    /// With the queue empty, a write naming a client id is sent live, where
464    /// a 404 for a note that landed as an older row is accepted; an hour
465    /// covers a session that saw the queue non-empty just before it emptied.
466    fn prune(&mut self, now: DateTime<Utc>) -> bool {
467        let before = self.0.len();
468        self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
469        self.0.len() != before
470    }
471
472    fn save(&self, dir: &Path) -> anyhow::Result<()> {
473        let path = dir.join(ALIASES_FILE);
474        if self.0.is_empty() {
475            return match std::fs::remove_file(&path) {
476                Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
477                _ => Ok(()),
478            };
479        }
480        let tmp = dir.join(format!(
481            "{ALIASES_FILE}.{}.tmp",
482            uuid::Uuid::new_v4().simple()
483        ));
484        let mut file = std::fs::OpenOptions::new()
485            .write(true)
486            .create_new(true)
487            .mode(0o600)
488            .open(&tmp)?;
489        file.write_all(&serde_json::to_vec(self)?)?;
490        file.sync_all()?;
491        std::fs::rename(&tmp, path)?;
492        Ok(())
493    }
494}
495
496/// `Ok(None)` when another replayer or the MCP server's ingest collapse
497/// removed it after the listing.
498fn read_entry(path: &Path, identity: Option<&Identity>) -> Result<Option<Entry>, OpenError> {
499    match std::fs::read(path) {
500        Ok(bytes) => open(&bytes, identity).map(Some),
501        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
502        Err(e) => Err(OpenError::Unreadable(e.into())),
503    }
504}
505
506/// Renamed out of the entry namespace rather than deleted, so the write it
507/// held can still be read by hand.
508fn set_aside(path: &Path, error: &anyhow::Error) {
509    let aside = path.with_extension("unreadable");
510    tracing::warn!(
511        "write queue entry {} is unreadable ({error:#}); moved to {}",
512        path.display(),
513        aside.display()
514    );
515    if let Err(e) = std::fs::rename(path, &aside) {
516        tracing::warn!("cannot move {} aside: {e}", path.display());
517    }
518}
519
520fn remove_entry(path: &Path) {
521    if let Err(e) = std::fs::remove_file(path)
522        && e.kind() != std::io::ErrorKind::NotFound
523    {
524        tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
525    }
526}
527
528fn replace_strings(value: &mut Value, from: &str, to: &str) {
529    match value {
530        Value::String(s) if s.contains(from) => *s = s.replace(from, to),
531        Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
532        Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
533        _ => {}
534    }
535}
536
537#[cfg(test)]
538mod tests {
539    use super::*;
540
541    #[test]
542    fn the_device_identity_is_published_for_the_mcp_server() {
543        let dir = tempfile::tempdir().unwrap();
544        assert!(publish_device_identity(dir.path(), &KeyStorage::File).is_none());
545        assert!(!dir.path().join(keys::DEVICE_PUB_FILE).exists());
546        let device = keys::create_device_keys(dir.path(), &KeyStorage::File).unwrap();
547        let identity = publish_device_identity(dir.path(), &KeyStorage::File).unwrap();
548        assert_eq!(identity.to_public(), device.age_recipient());
549        let published = std::fs::read_to_string(dir.path().join(keys::DEVICE_PUB_FILE)).unwrap();
550        assert_eq!(published.trim(), device.age_recipient().to_string());
551    }
552
553    #[test]
554    fn an_entry_name_dates_the_call() {
555        let at = entry_time("1793505622000000000-x.json").unwrap();
556        assert_eq!(at.timestamp(), 1_793_505_622);
557    }
558
559    #[test]
560    fn only_a_hard_forget_treats_404_as_done() {
561        let forget = |query: &str| Entry {
562            tool: "forget".to_string(),
563            method: "DELETE".to_string(),
564            path: "/v1/memory/x".to_string(),
565            query: Some(query.to_string()),
566            body: None,
567        };
568        assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
569        assert!(!is_hard_forget(&forget("hard=false")));
570        assert!(is_idempotent_removal(&forget("hard=true")));
571        assert!(!is_idempotent_removal(&forget("hard=false")));
572    }
573
574    #[test]
575    fn a_replayed_delete_treats_404_as_done_and_other_posts_do_not() {
576        let post = |path: &str| Entry {
577            tool: "delete".to_string(),
578            method: "POST".to_string(),
579            path: path.to_string(),
580            query: None,
581            body: Some(serde_json::json!({"session": "s1"})),
582        };
583        assert!(is_idempotent_removal(&post("/v1/delete")));
584        assert!(!is_idempotent_removal(&post("/v1/remember")));
585    }
586}