1use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Write;
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Path, PathBuf};
11use std::sync::Arc;
12use std::time::{Duration, SystemTime};
13
14use axum::Router;
15use axum::body::Body;
16use axum::http::{Method, Request, StatusCode, header};
17use chrono::{DateTime, Utc};
18use fs4::fs_std::FileExt;
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use tower::ServiceExt;
22
23use crate::health::Health;
24use crate::write_gate::{REPLAY_HEADER, WriteGate};
25
26pub const DIR_NAME: &str = "queue";
27pub const INTERVAL: Duration = Duration::from_secs(30);
28const LOCK_FILE: &str = ".lock";
29const STALE_TMP: Duration = Duration::from_secs(60 * 60);
32const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
33const ALIASES_FILE: &str = "aliases.json";
34const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
35const MAX_ENTRIES: usize = 10_000;
37
38#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
40pub struct Entry {
41 pub tool: String,
42 pub method: String,
43 pub path: String,
44 #[serde(default)]
46 pub query: Option<String>,
47 #[serde(default)]
48 pub body: Option<Value>,
49}
50
51pub fn is_entry_name(name: &str) -> bool {
54 let Some(stem) = name.strip_suffix(".json") else {
55 return false;
56 };
57 let Some((nanos, id)) = stem.split_at_checked(19) else {
58 return false;
59 };
60 nanos.bytes().all(|b| b.is_ascii_digit())
61 && id
62 .strip_prefix('-')
63 .is_some_and(|id| id.len() == 36 && uuid::Uuid::parse_str(id).is_ok())
64}
65
66fn entry_time(name: &str) -> Option<DateTime<Utc>> {
67 let nanos: i64 = name.get(..19)?.parse().ok()?;
68 Some(DateTime::from_timestamp_nanos(nanos))
69}
70
71pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
73 let mut names = Vec::new();
74 for item in std::fs::read_dir(dir)? {
75 let name = item?.file_name().to_string_lossy().into_owned();
76 if is_entry_name(&name) {
77 names.push(name);
78 }
79 }
80 names.sort();
81 Ok(names)
82}
83
84pub fn push(dir: &Path, entry: &Entry) -> anyhow::Result<()> {
87 std::fs::DirBuilder::new()
88 .recursive(true)
89 .mode(0o700)
90 .create(dir)?;
91 match entry_names(dir) {
92 Ok(names) if names.len() >= MAX_ENTRIES => {
93 anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
94 }
95 Ok(_) => {}
96 Err(e) => return Err(e.into()),
97 }
98 let nanos = SystemTime::now()
99 .duration_since(SystemTime::UNIX_EPOCH)?
100 .as_nanos();
101 let name = format!("{nanos:019}-{}.json", uuid::Uuid::new_v4());
102 let tmp = dir.join(format!("{name}.tmp"));
103 let mut file = std::fs::OpenOptions::new()
104 .write(true)
105 .create_new(true)
106 .mode(0o600)
107 .open(&tmp)?;
108 file.write_all(&serde_json::to_vec(entry)?)?;
109 file.sync_all()?;
110 std::fs::rename(&tmp, dir.join(name))?;
111 Ok(())
112}
113
114#[derive(Debug, Default, PartialEq)]
115pub struct Pass {
116 pub sent: usize,
117 pub rejected: usize,
118 pub stopped: Option<String>,
120}
121
122pub struct WriteQueue {
123 dir: PathBuf,
124 app: Router,
125 token: String,
126 health: Arc<Health>,
127 gate: Arc<WriteGate>,
128}
129
130enum Verdict {
131 Done,
132 Rejected(StatusCode, String),
133 Retry(String),
134}
135
136impl WriteQueue {
137 pub fn new(
138 dir: PathBuf,
139 app: Router,
140 token: String,
141 health: Arc<Health>,
142 gate: Arc<WriteGate>,
143 ) -> Self {
144 Self {
145 dir,
146 app,
147 token,
148 health,
149 gate,
150 }
151 }
152
153 pub fn spawn(self) {
154 tokio::spawn(async move {
155 let mut interval = tokio::time::interval(INTERVAL);
156 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
157 loop {
158 tokio::select! {
159 _ = interval.tick() => {}
160 _ = self.gate.reopened() => {}
161 }
162 self.tick().await;
163 }
164 });
165 }
166
167 pub async fn tick(&self) -> Option<Pass> {
171 if self.gate.is_closed() {
172 self.report();
173 return None;
174 }
175 let pass = match self.try_lock() {
176 Ok(Some(_lock)) => {
177 self.remove_stale_tmp();
178 Some(self.replay().await)
179 }
180 Ok(None) => None,
181 Err(e) => {
182 tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
183 None
184 }
185 };
186 self.gate.open_when_drained(|| self.is_empty());
187 self.report();
188 pass
189 }
190
191 fn is_empty(&self) -> bool {
192 match entry_names(&self.dir) {
193 Ok(names) => names.is_empty(),
194 Err(e) => e.kind() == std::io::ErrorKind::NotFound,
195 }
196 }
197
198 fn try_lock(&self) -> std::io::Result<Option<File>> {
199 if !self.dir.is_dir() {
200 return Ok(None);
201 }
202 let file = std::fs::OpenOptions::new()
203 .create(true)
204 .truncate(false)
205 .write(true)
206 .open(self.dir.join(LOCK_FILE))?;
207 Ok(file.try_lock_exclusive()?.then_some(file))
208 }
209
210 fn report(&self) {
211 let names = match entry_names(&self.dir) {
212 Ok(names) => names,
213 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
214 Err(e) => {
215 tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
216 return;
217 }
218 };
219 let oldest = names.first().and_then(|n| entry_time(n));
220 self.health
221 .queue_checked(names.len() as u64, oldest, Utc::now());
222 }
223
224 fn remove_stale_tmp(&self) {
225 let Ok(items) = std::fs::read_dir(&self.dir) else {
226 return;
227 };
228 let now = SystemTime::now();
229 for item in items.flatten() {
230 let path = item.path();
231 let stale = path.extension().is_some_and(|e| e == "tmp")
232 && item
233 .metadata()
234 .and_then(|m| m.modified())
235 .is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
236 if stale && let Err(e) = std::fs::remove_file(&path) {
237 tracing::warn!("cannot remove {}: {e}", path.display());
238 }
239 }
240 }
241
242 async fn replay(&self) -> Pass {
243 let mut pass = Pass::default();
244 let names = match entry_names(&self.dir) {
245 Ok(names) => names,
246 Err(e) => {
247 pass.stopped = Some(format!("listing the queue: {e}"));
248 return pass;
249 }
250 };
251 let mut aliases = Aliases::load(&self.dir);
252 for name in &names {
253 let path = self.dir.join(name);
254 let entry = match read_entry(&path) {
255 Ok(Some(entry)) => aliases.apply(entry),
256 Ok(None) => continue,
257 Err(e) => {
258 set_aside(&path, &e);
259 continue;
260 }
261 };
262 let (verdict, reply) = self.send(&entry).await;
263 match verdict {
264 Verdict::Done => {
265 if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
266 aliases.add(client, stored, Utc::now());
267 if let Err(e) = aliases.save(&self.dir) {
268 pass.stopped = Some(format!("saving the id aliases: {e:#}"));
269 return pass;
270 }
271 }
272 remove_entry(&path);
273 pass.sent += 1;
274 }
275 Verdict::Rejected(status, message) => {
276 tracing::warn!(
277 "queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
278 entry.tool,
279 status.as_u16()
280 );
281 self.health
282 .write_rejected(&entry.tool, status.as_u16(), Utc::now());
283 remove_entry(&path);
284 pass.rejected += 1;
285 }
286 Verdict::Retry(why) => {
287 tracing::info!("write queue replay paused at {name}: {why}");
288 pass.stopped = Some(why);
289 return pass;
290 }
291 }
292 }
293 if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
294 && aliases.prune(Utc::now())
295 && let Err(e) = aliases.save(&self.dir)
296 {
297 tracing::warn!("cannot prune the write queue id aliases: {e:#}");
298 }
299 pass
300 }
301
302 async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
303 let request = match build_request(entry, &self.token) {
304 Ok(r) => r,
305 Err(e) => {
306 return (
307 Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
308 None,
309 );
310 }
311 };
312 let response = match self.app.clone().oneshot(request).await {
313 Ok(r) => r,
314 Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
315 };
316 let status = response.status();
317 let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
318 Ok(b) => b,
319 Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
320 };
321 let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
322 let verdict =
323 if status.is_success() || (status == StatusCode::NOT_FOUND && is_hard_forget(entry)) {
324 Verdict::Done
325 } else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
326 Verdict::Retry(format!("HTTP {status}"))
327 } else {
328 let message = reply
329 .as_ref()
330 .and_then(|v| v["error"].as_str().map(str::to_string))
331 .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
332 Verdict::Rejected(status, message)
333 };
334 (verdict, reply)
335 }
336}
337
338fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
339 let method = Method::from_bytes(entry.method.as_bytes())?;
340 let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
341 Some(q) => format!("{}?{q}", entry.path),
342 None => entry.path.clone(),
343 };
344 let builder = Request::builder()
345 .method(method)
346 .uri(uri)
347 .header(header::AUTHORIZATION, format!("Bearer {token}"))
348 .header(REPLAY_HEADER, "1");
349 Ok(match &entry.body {
350 Some(body) => builder
351 .header(header::CONTENT_TYPE, "application/json")
352 .body(Body::from(body.to_string()))?,
353 None => builder.body(Body::empty())?,
354 })
355}
356
357fn is_hard_forget(entry: &Entry) -> bool {
358 entry.method.eq_ignore_ascii_case("DELETE")
359 && entry
360 .query
361 .as_deref()
362 .is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
363}
364
365fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
368 if entry.tool != "remember" {
369 return None;
370 }
371 let sent = entry.body.as_ref()?["id"].as_str()?;
372 let stored = reply?["id"].as_str()?;
373 (sent != stored).then(|| (sent.to_string(), stored.to_string()))
374}
375
376#[derive(Default, Serialize, Deserialize)]
381struct Aliases(BTreeMap<String, Alias>);
382
383#[derive(Serialize, Deserialize)]
384struct Alias {
385 to: String,
386 at: DateTime<Utc>,
387}
388
389impl Aliases {
390 fn load(dir: &Path) -> Self {
391 match std::fs::read(dir.join(ALIASES_FILE)) {
392 Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
393 tracing::warn!("ignoring unreadable write queue id aliases: {e}");
394 Self::default()
395 }),
396 Err(_) => Self::default(),
397 }
398 }
399
400 fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
401 self.0.insert(
402 client,
403 Alias {
404 to: stored,
405 at: now,
406 },
407 );
408 }
409
410 fn apply(&self, mut entry: Entry) -> Entry {
411 for (client, alias) in &self.0 {
412 entry.path = entry.path.replace(client.as_str(), &alias.to);
413 entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
414 if let Some(body) = entry.body.as_mut() {
415 replace_strings(body, client, &alias.to);
416 }
417 }
418 entry
419 }
420
421 fn prune(&mut self, now: DateTime<Utc>) -> bool {
425 let before = self.0.len();
426 self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
427 self.0.len() != before
428 }
429
430 fn save(&self, dir: &Path) -> anyhow::Result<()> {
431 let path = dir.join(ALIASES_FILE);
432 if self.0.is_empty() {
433 return match std::fs::remove_file(&path) {
434 Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
435 _ => Ok(()),
436 };
437 }
438 let tmp = dir.join(format!(
439 "{ALIASES_FILE}.{}.tmp",
440 uuid::Uuid::new_v4().simple()
441 ));
442 let mut file = std::fs::OpenOptions::new()
443 .write(true)
444 .create_new(true)
445 .mode(0o600)
446 .open(&tmp)?;
447 file.write_all(&serde_json::to_vec(self)?)?;
448 file.sync_all()?;
449 std::fs::rename(&tmp, path)?;
450 Ok(())
451 }
452}
453
454fn read_entry(path: &Path) -> anyhow::Result<Option<Entry>> {
457 match std::fs::read(path) {
458 Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
459 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
460 Err(e) => Err(e.into()),
461 }
462}
463
464fn set_aside(path: &Path, error: &anyhow::Error) {
467 let aside = path.with_extension("unreadable");
468 tracing::warn!(
469 "write queue entry {} is unreadable ({error:#}); moved to {}",
470 path.display(),
471 aside.display()
472 );
473 if let Err(e) = std::fs::rename(path, &aside) {
474 tracing::warn!("cannot move {} aside: {e}", path.display());
475 }
476}
477
478fn remove_entry(path: &Path) {
479 if let Err(e) = std::fs::remove_file(path)
480 && e.kind() != std::io::ErrorKind::NotFound
481 {
482 tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
483 }
484}
485
486fn replace_strings(value: &mut Value, from: &str, to: &str) {
487 match value {
488 Value::String(s) if s.contains(from) => *s = s.replace(from, to),
489 Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
490 Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
491 _ => {}
492 }
493}
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498
499 #[test]
500 fn only_timestamped_uuid_json_names_are_entries() {
501 let id = "0b5c2f9e-5d0e-4c1a-9a57-0c7f4f1f2a10";
502 assert!(is_entry_name(&format!("1793505622000000000-{id}.json")));
503 for name in [
504 "projects.json".to_string(),
505 ".lock".to_string(),
506 format!("1793505622000000000-{id}.json.tmp"),
507 format!("179350562200000000-{id}.json"),
508 format!("1793505622000000000-{id}.unreadable"),
509 "1793505622000000000-not-a-uuid.json".to_string(),
510 ] {
511 assert!(!is_entry_name(&name), "{name}");
512 }
513 }
514
515 #[test]
516 fn an_entry_name_dates_the_call() {
517 let at = entry_time("1793505622000000000-x.json").unwrap();
518 assert_eq!(at.timestamp(), 1_793_505_622);
519 }
520
521 #[test]
522 fn only_a_hard_forget_treats_404_as_done() {
523 let forget = |query: &str| Entry {
524 tool: "forget".to_string(),
525 method: "DELETE".to_string(),
526 path: "/v1/memory/x".to_string(),
527 query: Some(query.to_string()),
528 body: None,
529 };
530 assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
531 assert!(!is_hard_forget(&forget("hard=false")));
532 }
533}