1use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::time::{Duration, SystemTime};
13
14use axum::Router;
15use axum::body::Body;
16use axum::http::{Method, Request, StatusCode, header};
17use chrono::{DateTime, Utc};
18use fs4::fs_std::FileExt;
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use tower::ServiceExt;
22
23use crate::health::Health;
24use crate::write_gate::{REPLAY_HEADER, WriteGate};
25
26pub use second_brain_core::write_queue::{DIR_NAME, is_entry_name};
27pub const INTERVAL: Duration = Duration::from_secs(30);
28const LOCK_FILE: &str = ".lock";
29const STALE_TMP: Duration = Duration::from_secs(60 * 60);
32const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
33const ALIASES_FILE: &str = "aliases.json";
34const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
35const MAX_ENTRIES: usize = 10_000;
37
38#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
40pub struct Entry {
41 pub tool: String,
42 pub method: String,
43 pub path: String,
44 #[serde(default)]
46 pub query: Option<String>,
47 #[serde(default)]
48 pub body: Option<Value>,
49}
50
51fn entry_time(name: &str) -> Option<DateTime<Utc>> {
52 let nanos: i64 = name.get(..19)?.parse().ok()?;
53 Some(DateTime::from_timestamp_nanos(nanos))
54}
55
56pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
58 let mut names = Vec::new();
59 for item in std::fs::read_dir(dir)? {
60 let name = item?.file_name().to_string_lossy().into_owned();
61 if is_entry_name(&name) {
62 names.push(name);
63 }
64 }
65 names.sort();
66 Ok(names)
67}
68
69pub fn push(dir: &Path, entry: &Entry) -> anyhow::Result<()> {
72 std::fs::DirBuilder::new()
73 .recursive(true)
74 .mode(0o700)
75 .create(dir)?;
76 match entry_names(dir) {
77 Ok(names) if names.len() >= MAX_ENTRIES => {
78 anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
79 }
80 Ok(_) => {}
81 Err(e) => return Err(e.into()),
82 }
83 let nanos = SystemTime::now()
84 .duration_since(SystemTime::UNIX_EPOCH)?
85 .as_nanos();
86 let name = format!("{nanos:019}-{}.json", uuid::Uuid::new_v4());
87 let tmp = dir.join(format!("{name}.tmp"));
88 let mut file = std::fs::OpenOptions::new()
89 .write(true)
90 .create_new(true)
91 .mode(0o600)
92 .open(&tmp)?;
93 file.write_all(&serde_json::to_vec(entry)?)?;
94 file.sync_all()?;
95 std::fs::rename(&tmp, dir.join(name))?;
96 Ok(())
97}
98
99#[derive(Debug, Default, PartialEq)]
100pub struct Pass {
101 pub sent: usize,
102 pub rejected: usize,
103 pub stopped: Option<String>,
105}
106
107pub struct WriteQueue {
108 dir: PathBuf,
109 app: Router,
110 token: String,
111 health: Arc<Health>,
112 gate: Arc<WriteGate>,
113}
114
115enum Verdict {
116 Done,
117 Rejected(StatusCode, String),
118 Retry(String),
119}
120
121impl WriteQueue {
122 pub fn new(
123 dir: PathBuf,
124 app: Router,
125 token: String,
126 health: Arc<Health>,
127 gate: Arc<WriteGate>,
128 ) -> Self {
129 Self {
130 dir,
131 app,
132 token,
133 health,
134 gate,
135 }
136 }
137
138 pub fn spawn(self) {
139 tokio::spawn(async move {
140 let mut interval = tokio::time::interval(INTERVAL);
141 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
142 loop {
143 tokio::select! {
144 _ = interval.tick() => {}
145 _ = self.gate.reopened() => {}
146 }
147 self.tick().await;
148 }
149 });
150 }
151
152 pub async fn tick(&self) -> Option<Pass> {
156 if self.gate.is_closed() {
157 self.report();
158 return None;
159 }
160 let pass = match self.try_lock() {
161 Ok(Some(_lock)) => {
162 self.remove_stale_tmp();
163 Some(self.replay().await)
164 }
165 Ok(None) => None,
166 Err(e) => {
167 tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
168 None
169 }
170 };
171 self.gate.open_when_drained(|| self.is_empty());
172 self.report();
173 pass
174 }
175
176 fn is_empty(&self) -> bool {
177 match entry_names(&self.dir) {
178 Ok(names) => names.is_empty(),
179 Err(e) => e.kind() == std::io::ErrorKind::NotFound,
180 }
181 }
182
183 fn try_lock(&self) -> std::io::Result<Option<File>> {
184 if !self.dir.is_dir() {
185 return Ok(None);
186 }
187 let file = std::fs::OpenOptions::new()
188 .create(true)
189 .truncate(false)
190 .write(true)
191 .open(self.dir.join(LOCK_FILE))?;
192 Ok(file.try_lock_exclusive()?.then_some(file))
193 }
194
195 fn report(&self) {
196 let names = match entry_names(&self.dir) {
197 Ok(names) => names,
198 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
199 Err(e) => {
200 tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
201 return;
202 }
203 };
204 let oldest = names.first().and_then(|n| entry_time(n));
205 self.health
206 .queue_checked(names.len() as u64, oldest, Utc::now());
207 }
208
209 fn remove_stale_tmp(&self) {
210 let Ok(items) = std::fs::read_dir(&self.dir) else {
211 return;
212 };
213 let now = SystemTime::now();
214 for item in items.flatten() {
215 let path = item.path();
216 let stale = path.extension().is_some_and(|e| e == "tmp")
217 && item
218 .metadata()
219 .and_then(|m| m.modified())
220 .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
221 if stale && let Err(e) = std::fs::remove_file(&path) {
222 tracing::warn!("cannot remove {}: {e}", path.display());
223 }
224 }
225 }
226
227 async fn replay(&self) -> Pass {
228 let mut pass = Pass::default();
229 let names = match entry_names(&self.dir) {
230 Ok(names) => names,
231 Err(e) => {
232 pass.stopped = Some(format!("listing the queue: {e}"));
233 return pass;
234 }
235 };
236 let mut aliases = Aliases::load(&self.dir);
237 for name in &names {
238 let path = self.dir.join(name);
239 let entry = match read_entry(&path) {
240 Ok(Some(entry)) => aliases.apply(entry),
241 Ok(None) => continue,
242 Err(e) => {
243 set_aside(&path, &e);
244 continue;
245 }
246 };
247 let (verdict, reply) = self.send(&entry).await;
248 match verdict {
249 Verdict::Done => {
250 if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
251 aliases.add(client, stored, Utc::now());
252 if let Err(e) = aliases.save(&self.dir) {
253 pass.stopped = Some(format!("saving the id aliases: {e:#}"));
254 return pass;
255 }
256 }
257 remove_entry(&path);
258 pass.sent += 1;
259 }
260 Verdict::Rejected(status, message) => {
261 tracing::warn!(
262 "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
263 entry.tool,
264 status.as_u16()
265 );
266 self.health
267 .write_rejected(&entry.tool, status.as_u16(), Utc::now());
268 remove_entry(&path);
269 pass.rejected += 1;
270 }
271 Verdict::Retry(why) => {
272 tracing::info!("write queue replay paused at {name}: {why}");
273 pass.stopped = Some(why);
274 return pass;
275 }
276 }
277 }
278 if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
279 && aliases.prune(Utc::now())
280 && let Err(e) = aliases.save(&self.dir)
281 {
282 tracing::warn!("cannot prune the write queue id aliases: {e:#}");
283 }
284 pass
285 }
286
287 async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
288 let request = match build_request(entry, &self.token) {
289 Ok(r) => r,
290 Err(e) => {
291 return (
292 Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
293 None,
294 );
295 }
296 };
297 let response = match self.app.clone().oneshot(request).await {
298 Ok(r) => r,
299 Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
300 };
301 let status = response.status();
302 let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
303 Ok(b) => b,
304 Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
305 };
306 let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
307 let verdict =
308 if status.is_success() || (status == StatusCode::NOT_FOUND && is_hard_forget(entry)) {
309 Verdict::Done
310 } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
311 Verdict::Retry(format!("HTTP {status}"))
312 } else {
313 let message = reply
314 .as_ref()
315 .and_then(|v| v["error"].as_str().map(str::to_string))
316 .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
317 Verdict::Rejected(status, message)
318 };
319 (verdict, reply)
320 }
321}
322
323fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
324 let method = Method::from_bytes(entry.method.as_bytes())?;
325 let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
326 Some(q) => format!("{}?{q}", entry.path),
327 None => entry.path.clone(),
328 };
329 let builder = Request::builder()
330 .method(method)
331 .uri(uri)
332 .header(header::AUTHORIZATION, format!("Bearer {token}"))
333 .header(REPLAY_HEADER, "1");
334 Ok(match &entry.body {
335 Some(body) => builder
336 .header(header::CONTENT_TYPE, "application/json")
337 .body(Body::from(body.to_string()))?,
338 None => builder.body(Body::empty())?,
339 })
340}
341
342fn is_hard_forget(entry: &Entry) -> bool {
343 entry.method.eq_ignore_ascii_case("DELETE")
344 && entry
345 .query
346 .as_deref()
347 .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
348}
349
350fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
353 if entry.tool != "remember" {
354 return None;
355 }
356 let sent = entry.body.as_ref()?["id"].as_str()?;
357 let stored = reply?["id"].as_str()?;
358 (sent != stored).then(|| (sent.to_string(), stored.to_string()))
359}
360
361#[derive(Default, Serialize, Deserialize)]
366struct Aliases(BTreeMap<String, Alias>);
367
368#[derive(Serialize, Deserialize)]
369struct Alias {
370 to: String,
371 at: DateTime<Utc>,
372}
373
374impl Aliases {
375 fn load(dir: &Path) -> Self {
376 match std::fs::read(dir.join(ALIASES_FILE)) {
377 Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
378 tracing::warn!("ignoring unreadable write queue id aliases: {e}");
379 Self::default()
380 }),
381 Err(_) => Self::default(),
382 }
383 }
384
385 fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
386 self.0.insert(
387 client,
388 Alias {
389 to: stored,
390 at: now,
391 },
392 );
393 }
394
395 fn apply(&self, mut entry: Entry) -> Entry {
396 for (client, alias) in &self.0 {
397 entry.path = entry.path.replace(client.as_str(), &alias.to);
398 entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
399 if let Some(body) = entry.body.as_mut() {
400 replace_strings(body, client, &alias.to);
401 }
402 }
403 entry
404 }
405
406 fn prune(&mut self, now: DateTime<Utc>) -> bool {
410 let before = self.0.len();
411 self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
412 self.0.len() != before
413 }
414
415 fn save(&self, dir: &Path) -> anyhow::Result<()> {
416 let path = dir.join(ALIASES_FILE);
417 if self.0.is_empty() {
418 return match std::fs::remove_file(&path) {
419 Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
420 _ => Ok(()),
421 };
422 }
423 let tmp = dir.join(format!(
424 "{ALIASES_FILE}.{}.tmp",
425 uuid::Uuid::new_v4().simple()
426 ));
427 let mut file = std::fs::OpenOptions::new()
428 .write(true)
429 .create_new(true)
430 .mode(0o600)
431 .open(&tmp)?;
432 file.write_all(&serde_json::to_vec(self)?)?;
433 file.sync_all()?;
434 std::fs::rename(&tmp, path)?;
435 Ok(())
436 }
437}
438
439fn read_entry(path: &Path) -> anyhow::Result<Option<Entry>> {
442 match std::fs::read(path) {
443 Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
444 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
445 Err(e) => Err(e.into()),
446 }
447}
448
449fn set_aside(path: &Path, error: &anyhow::Error) {
452 let aside = path.with_extension("unreadable");
453 tracing::warn!(
454 "write queue entry {} is unreadable ({error:#}); moved to {}",
455 path.display(),
456 aside.display()
457 );
458 if let Err(e) = std::fs::rename(path, &aside) {
459 tracing::warn!("cannot move {} aside: {e}", path.display());
460 }
461}
462
463fn remove_entry(path: &Path) {
464 if let Err(e) = std::fs::remove_file(path)
465 && e.kind() != std::io::ErrorKind::NotFound
466 {
467 tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
468 }
469}
470
471fn replace_strings(value: &mut Value, from: &str, to: &str) {
472 match value {
473 Value::String(s) if s.contains(from) => *s = s.replace(from, to),
474 Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
475 Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
476 _ => {}
477 }
478}
479
480#[cfg(test)]
481mod tests {
482 use super::*;
483
484 #[test]
485 fn an_entry_name_dates_the_call() {
486 let at = entry_time("1793505622000000000-x.json").unwrap();
487 assert_eq!(at.timestamp(), 1_793_505_622);
488 }
489
490 #[test]
491 fn only_a_hard_forget_treats_404_as_done() {
492 let forget = |query: &str| Entry {
493 tool: "forget".to_string(),
494 method: "DELETE".to_string(),
495 path: "/v1/memory/x".to_string(),
496 query: Some(query.to_string()),
497 body: None,
498 };
499 assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
500 assert!(!is_hard_forget(&forget("hard=false")));
501 }
502}