1use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::{Duration, SystemTime};
14
15use age::x25519::{Identity, Recipient};
16use axum::Router;
17use axum::body::Body;
18use axum::http::{Method, Request, StatusCode, header};
19use chrono::{DateTime, Utc};
20use fs4::fs_std::FileExt;
21use second_brain_core::keys::{self, KeyStorage};
22use second_brain_core::write_queue::{OpenError, entry_name, is_sealed_name, open};
23use serde::{Deserialize, Serialize};
24use serde_json::Value;
25use tower::ServiceExt;
26
27use crate::health::Health;
28use crate::write_gate::{REPLAY_HEADER, WriteGate};
29
30pub use second_brain_core::write_queue::{DIR_NAME, Entry, is_entry_name};
31pub const INTERVAL: Duration = Duration::from_secs(30);
32const LOCK_FILE: &str = ".lock";
33const STALE_TMP: Duration = Duration::from_secs(60 * 60);
36const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
37const ALIASES_FILE: &str = "aliases.json";
38const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
39const MAX_ENTRIES: usize = 10_000;
41
42fn entry_time(name: &str) -> Option<DateTime<Utc>> {
43 let nanos: i64 = name.get(..19)?.parse().ok()?;
44 Some(DateTime::from_timestamp_nanos(nanos))
45}
46
47pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
49 let mut names = Vec::new();
50 for item in std::fs::read_dir(dir)? {
51 let name = item?.file_name().to_string_lossy().into_owned();
52 if is_entry_name(&name) {
53 names.push(name);
54 }
55 }
56 names.sort();
57 Ok(names)
58}
59
60pub fn publish_device_identity(data_dir: &Path, storage: &KeyStorage) -> Option<Identity> {
64 let keys = match keys::load_device_keys_in(data_dir, storage) {
65 Ok(Some(keys)) => keys,
66 Ok(None) => {
67 tracing::warn!("no device key: queued writes are not sealed");
68 return None;
69 }
70 Err(e) => {
71 tracing::error!("device keys unreadable: queued writes are not sealed: {e:#}");
72 return None;
73 }
74 };
75 if let Err(e) = keys::publish_device_pub(data_dir, &keys) {
76 tracing::warn!("the MCP server cannot seal the writes it queues: {e:#}");
77 }
78 Some(keys.age_identity().clone())
79}
80
81pub fn push(dir: &Path, entry: &Entry, recipient: Option<&Recipient>) -> anyhow::Result<()> {
85 std::fs::DirBuilder::new()
86 .recursive(true)
87 .mode(0o700)
88 .create(dir)?;
89 match entry_names(dir) {
90 Ok(names) if names.len() >= MAX_ENTRIES => {
91 anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
92 }
93 Ok(_) => {}
94 Err(e) => return Err(e.into()),
95 }
96 let nanos = SystemTime::now()
97 .duration_since(SystemTime::UNIX_EPOCH)?
98 .as_nanos();
99 let bytes = match recipient {
100 Some(recipient) => entry.to_sealed_file(std::slice::from_ref(recipient))?,
101 None => entry.to_plain_file()?,
102 };
103 let name = entry_name(nanos, recipient.is_some());
104 let tmp = dir.join(format!("{name}.tmp"));
105 let mut file = std::fs::OpenOptions::new()
106 .write(true)
107 .create_new(true)
108 .mode(0o600)
109 .open(&tmp)?;
110 file.write_all(&bytes)?;
111 file.sync_all()?;
112 std::fs::rename(&tmp, dir.join(name))?;
113 Ok(())
114}
115
116#[derive(Debug, Default, PartialEq)]
117pub struct Pass {
118 pub sent: usize,
119 pub rejected: usize,
120 pub waiting: usize,
122 pub stopped: Option<String>,
124}
125
126pub struct WriteQueue {
127 dir: PathBuf,
128 app: Router,
129 token: String,
130 health: Arc<Health>,
131 gate: Arc<WriteGate>,
132 identity: Option<Identity>,
133 warned_no_key: AtomicBool,
134}
135
136enum Verdict {
137 Done,
138 Rejected(StatusCode, String),
139 Retry(String),
140}
141
142impl WriteQueue {
143 pub fn new(
144 dir: PathBuf,
145 app: Router,
146 token: String,
147 health: Arc<Health>,
148 gate: Arc<WriteGate>,
149 ) -> Self {
150 Self {
151 dir,
152 app,
153 token,
154 health,
155 gate,
156 identity: None,
157 warned_no_key: AtomicBool::new(false),
158 }
159 }
160
161 pub fn opening_with(mut self, identity: Identity) -> Self {
164 self.identity = Some(identity);
165 self
166 }
167
168 pub fn spawn(self) {
169 tokio::spawn(async move {
170 let mut interval = tokio::time::interval(INTERVAL);
171 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
172 loop {
173 tokio::select! {
174 _ = interval.tick() => {}
175 _ = self.gate.reopened() => {}
176 }
177 self.tick().await;
178 }
179 });
180 }
181
182 pub async fn tick(&self) -> Option<Pass> {
186 if self.gate.is_closed() {
187 self.report();
188 return None;
189 }
190 let pass = match self.try_lock() {
191 Ok(Some(_lock)) => {
192 self.remove_stale_tmp();
193 Some(self.replay().await)
194 }
195 Ok(None) => None,
196 Err(e) => {
197 tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
198 None
199 }
200 };
201 self.gate.open_when_drained(|| self.nothing_to_replay());
202 self.report();
203 pass
204 }
205
206 fn nothing_to_replay(&self) -> bool {
209 match entry_names(&self.dir) {
210 Ok(names) => names.iter().all(|name| self.waits_for_key(name)),
211 Err(e) => e.kind() == std::io::ErrorKind::NotFound,
212 }
213 }
214
215 fn waits_for_key(&self, name: &str) -> bool {
216 self.identity.is_none() && is_sealed_name(name)
217 }
218
219 fn try_lock(&self) -> std::io::Result<Option<File>> {
220 if !self.dir.is_dir() {
221 return Ok(None);
222 }
223 let file = std::fs::OpenOptions::new()
224 .create(true)
225 .truncate(false)
226 .write(true)
227 .open(self.dir.join(LOCK_FILE))?;
228 Ok(file.try_lock_exclusive()?.then_some(file))
229 }
230
231 fn report(&self) {
232 let names = match entry_names(&self.dir) {
233 Ok(names) => names,
234 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
235 Err(e) => {
236 tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
237 return;
238 }
239 };
240 let oldest = names.first().and_then(|n| entry_time(n));
241 self.health
242 .queue_checked(names.len() as u64, oldest, Utc::now());
243 }
244
245 fn remove_stale_tmp(&self) {
246 let Ok(items) = std::fs::read_dir(&self.dir) else {
247 return;
248 };
249 let now = SystemTime::now();
250 for item in items.flatten() {
251 let path = item.path();
252 let stale = path.extension().is_some_and(|e| e == "tmp")
253 && item
254 .metadata()
255 .and_then(|m| m.modified())
256 .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
257 if stale && let Err(e) = std::fs::remove_file(&path) {
258 tracing::warn!("cannot remove {}: {e}", path.display());
259 }
260 }
261 }
262
263 async fn replay(&self) -> Pass {
264 let mut pass = Pass::default();
265 let names = match entry_names(&self.dir) {
266 Ok(names) => names,
267 Err(e) => {
268 pass.stopped = Some(format!("listing the queue: {e}"));
269 return pass;
270 }
271 };
272 let mut aliases = Aliases::load(&self.dir);
273 for name in &names {
274 let path = self.dir.join(name);
275 let entry = match read_entry(&path, self.identity.as_ref()) {
276 Ok(Some(entry)) => aliases.apply(entry),
277 Ok(None) => continue,
278 Err(OpenError::NoKey) => {
279 pass.waiting += 1;
280 continue;
281 }
282 Err(OpenError::Unreadable(e)) => {
283 set_aside(&path, &e);
284 continue;
285 }
286 };
287 let (verdict, reply) = self.send(&entry).await;
288 match verdict {
289 Verdict::Done => {
290 if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
291 aliases.add(client, stored, Utc::now());
292 if let Err(e) = aliases.save(&self.dir) {
293 pass.stopped = Some(format!("saving the id aliases: {e:#}"));
294 return pass;
295 }
296 }
297 remove_entry(&path);
298 pass.sent += 1;
299 }
300 Verdict::Rejected(status, message) => {
301 tracing::warn!(
302 "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
303 entry.tool,
304 status.as_u16()
305 );
306 self.health
307 .write_rejected(&entry.tool, status.as_u16(), Utc::now());
308 remove_entry(&path);
309 pass.rejected += 1;
310 }
311 Verdict::Retry(why) => {
312 tracing::info!("write queue replay paused at {name}: {why}");
313 pass.stopped = Some(why);
314 return pass;
315 }
316 }
317 }
318 if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
319 && aliases.prune(Utc::now())
320 && let Err(e) = aliases.save(&self.dir)
321 {
322 tracing::warn!("cannot prune the write queue id aliases: {e:#}");
323 }
324 if pass.waiting > 0 && !self.warned_no_key.swap(true, Ordering::Relaxed) {
325 tracing::warn!(
326 "{} sealed queued writes wait in {} for the device key, which this daemon \
327 could not read at start; later writes replay without them",
328 pass.waiting,
329 self.dir.display()
330 );
331 }
332 pass
333 }
334
335 async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
336 let request = match build_request(entry, &self.token) {
337 Ok(r) => r,
338 Err(e) => {
339 return (
340 Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
341 None,
342 );
343 }
344 };
345 let response = match self.app.clone().oneshot(request).await {
346 Ok(r) => r,
347 Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
348 };
349 let status = response.status();
350 let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
351 Ok(b) => b,
352 Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
353 };
354 let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
355 let verdict = if status.is_success()
356 || (status == StatusCode::NOT_FOUND && is_idempotent_removal(entry))
357 {
358 Verdict::Done
359 } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
360 Verdict::Retry(format!("HTTP {status}"))
361 } else {
362 let message = reply
363 .as_ref()
364 .and_then(|v| v["error"].as_str().map(str::to_string))
365 .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
366 Verdict::Rejected(status, message)
367 };
368 (verdict, reply)
369 }
370}
371
372fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
373 let method = Method::from_bytes(entry.method.as_bytes())?;
374 let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
375 Some(q) => format!("{}?{q}", entry.path),
376 None => entry.path.clone(),
377 };
378 let builder = Request::builder()
379 .method(method)
380 .uri(uri)
381 .header(header::AUTHORIZATION, format!("Bearer {token}"))
382 .header(REPLAY_HEADER, "1");
383 Ok(match &entry.body {
384 Some(body) => builder
385 .header(header::CONTENT_TYPE, "application/json")
386 .body(Body::from(body.to_string()))?,
387 None => builder.body(Body::empty())?,
388 })
389}
390
391fn is_idempotent_removal(entry: &Entry) -> bool {
392 is_hard_forget(entry) || is_delete(entry)
393}
394
395fn is_delete(entry: &Entry) -> bool {
396 entry.method.eq_ignore_ascii_case("POST") && entry.path == "/v1/delete"
397}
398
399fn is_hard_forget(entry: &Entry) -> bool {
400 entry.method.eq_ignore_ascii_case("DELETE")
401 && entry
402 .query
403 .as_deref()
404 .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
405}
406
407fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
410 if entry.tool != "remember" {
411 return None;
412 }
413 let sent = entry.body.as_ref()?["id"].as_str()?;
414 let stored = reply?["id"].as_str()?;
415 (sent != stored).then(|| (sent.to_string(), stored.to_string()))
416}
417
418#[derive(Default, Serialize, Deserialize)]
423struct Aliases(BTreeMap<String, Alias>);
424
425#[derive(Serialize, Deserialize)]
426struct Alias {
427 to: String,
428 at: DateTime<Utc>,
429}
430
431impl Aliases {
432 fn load(dir: &Path) -> Self {
433 match std::fs::read(dir.join(ALIASES_FILE)) {
434 Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
435 tracing::warn!("ignoring unreadable write queue id aliases: {e}");
436 Self::default()
437 }),
438 Err(_) => Self::default(),
439 }
440 }
441
442 fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
443 self.0.insert(
444 client,
445 Alias {
446 to: stored,
447 at: now,
448 },
449 );
450 }
451
452 fn apply(&self, mut entry: Entry) -> Entry {
453 for (client, alias) in &self.0 {
454 entry.path = entry.path.replace(client.as_str(), &alias.to);
455 entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
456 if let Some(body) = entry.body.as_mut() {
457 replace_strings(body, client, &alias.to);
458 }
459 }
460 entry
461 }
462
463 fn prune(&mut self, now: DateTime<Utc>) -> bool {
467 let before = self.0.len();
468 self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
469 self.0.len() != before
470 }
471
472 fn save(&self, dir: &Path) -> anyhow::Result<()> {
473 let path = dir.join(ALIASES_FILE);
474 if self.0.is_empty() {
475 return match std::fs::remove_file(&path) {
476 Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
477 _ => Ok(()),
478 };
479 }
480 let tmp = dir.join(format!(
481 "{ALIASES_FILE}.{}.tmp",
482 uuid::Uuid::new_v4().simple()
483 ));
484 let mut file = std::fs::OpenOptions::new()
485 .write(true)
486 .create_new(true)
487 .mode(0o600)
488 .open(&tmp)?;
489 file.write_all(&serde_json::to_vec(self)?)?;
490 file.sync_all()?;
491 std::fs::rename(&tmp, path)?;
492 Ok(())
493 }
494}
495
496fn read_entry(path: &Path, identity: Option<&Identity>) -> Result<Option<Entry>, OpenError> {
499 match std::fs::read(path) {
500 Ok(bytes) => open(&bytes, identity).map(Some),
501 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
502 Err(e) => Err(OpenError::Unreadable(e.into())),
503 }
504}
505
506fn set_aside(path: &Path, error: &anyhow::Error) {
509 let aside = path.with_extension("unreadable");
510 tracing::warn!(
511 "write queue entry {} is unreadable ({error:#}); moved to {}",
512 path.display(),
513 aside.display()
514 );
515 if let Err(e) = std::fs::rename(path, &aside) {
516 tracing::warn!("cannot move {} aside: {e}", path.display());
517 }
518}
519
520fn remove_entry(path: &Path) {
521 if let Err(e) = std::fs::remove_file(path)
522 && e.kind() != std::io::ErrorKind::NotFound
523 {
524 tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
525 }
526}
527
528fn replace_strings(value: &mut Value, from: &str, to: &str) {
529 match value {
530 Value::String(s) if s.contains(from) => *s = s.replace(from, to),
531 Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
532 Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
533 _ => {}
534 }
535}
536
537#[cfg(test)]
538mod tests {
539 use super::*;
540
541 #[test]
542 fn the_device_identity_is_published_for_the_mcp_server() {
543 let dir = tempfile::tempdir().unwrap();
544 assert!(publish_device_identity(dir.path(), &KeyStorage::File).is_none());
545 assert!(!dir.path().join(keys::DEVICE_PUB_FILE).exists());
546 let device = keys::create_device_keys(dir.path(), &KeyStorage::File).unwrap();
547 let identity = publish_device_identity(dir.path(), &KeyStorage::File).unwrap();
548 assert_eq!(identity.to_public(), device.age_recipient());
549 let published = std::fs::read_to_string(dir.path().join(keys::DEVICE_PUB_FILE)).unwrap();
550 assert_eq!(published.trim(), device.age_recipient().to_string());
551 }
552
553 #[test]
554 fn an_entry_name_dates_the_call() {
555 let at = entry_time("1793505622000000000-x.json").unwrap();
556 assert_eq!(at.timestamp(), 1_793_505_622);
557 }
558
559 #[test]
560 fn only_a_hard_forget_treats_404_as_done() {
561 let forget = |query: &str| Entry {
562 tool: "forget".to_string(),
563 method: "DELETE".to_string(),
564 path: "/v1/memory/x".to_string(),
565 query: Some(query.to_string()),
566 body: None,
567 };
568 assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
569 assert!(!is_hard_forget(&forget("hard=false")));
570 assert!(is_idempotent_removal(&forget("hard=true")));
571 assert!(!is_idempotent_removal(&forget("hard=false")));
572 }
573
574 #[test]
575 fn a_replayed_delete_treats_404_as_done_and_other_posts_do_not() {
576 let post = |path: &str| Entry {
577 tool: "delete".to_string(),
578 method: "POST".to_string(),
579 path: path.to_string(),
580 query: None,
581 body: Some(serde_json::json!({"session": "s1"})),
582 };
583 assert!(is_idempotent_removal(&post("/v1/delete")));
584 assert!(!is_idempotent_removal(&post("/v1/remember")));
585 }
586}