use std::collections::BTreeMap;
use std::fs::File;
use std::io::Write;
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use axum::Router;
use axum::body::Body;
use axum::http::{Method, Request, StatusCode, header};
use chrono::{DateTime, Utc};
use fs4::fs_std::FileExt;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tower::ServiceExt;
use crate::health::Health;
use crate::write_gate::{REPLAY_HEADER, WriteGate};
pub const DIR_NAME: &str = "queue";
pub const INTERVAL: Duration = Duration::from_secs(30);
const LOCK_FILE: &str = ".lock";
const STALE_TMP: Duration = Duration::from_secs(60 * 60);
const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
const ALIASES_FILE: &str = "aliases.json";
const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
const MAX_ENTRIES: usize = 10_000;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Entry {
pub tool: String,
pub method: String,
pub path: String,
#[serde(default)]
pub query: Option<String>,
#[serde(default)]
pub body: Option<Value>,
}
pub fn is_entry_name(name: &str) -> bool {
let Some(stem) = name.strip_suffix(".json") else {
return false;
};
let Some((nanos, id)) = stem.split_at_checked(19) else {
return false;
};
nanos.bytes().all(|b| b.is_ascii_digit())
&& id
.strip_prefix('-')
.is_some_and(|id| id.len() == 36 && uuid::Uuid::parse_str(id).is_ok())
}
fn entry_time(name: &str) -> Option<DateTime<Utc>> {
let nanos: i64 = name.get(..19)?.parse().ok()?;
Some(DateTime::from_timestamp_nanos(nanos))
}
pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
let mut names = Vec::new();
for item in std::fs::read_dir(dir)? {
let name = item?.file_name().to_string_lossy().into_owned();
if is_entry_name(&name) {
names.push(name);
}
}
names.sort();
Ok(names)
}
pub fn push(dir: &Path, entry: &Entry) -> anyhow::Result<()> {
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(dir)?;
match entry_names(dir) {
Ok(names) if names.len() >= MAX_ENTRIES => {
anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
}
Ok(_) => {}
Err(e) => return Err(e.into()),
}
let nanos = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)?
.as_nanos();
let name = format!("{nanos:019}-{}.json", uuid::Uuid::new_v4());
let tmp = dir.join(format!("{name}.tmp"));
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&tmp)?;
file.write_all(&serde_json::to_vec(entry)?)?;
file.sync_all()?;
std::fs::rename(&tmp, dir.join(name))?;
Ok(())
}
#[derive(Debug, Default, PartialEq)]
pub struct Pass {
pub sent: usize,
pub rejected: usize,
pub stopped: Option<String>,
}
pub struct WriteQueue {
dir: PathBuf,
app: Router,
token: String,
health: Arc<Health>,
gate: Arc<WriteGate>,
}
enum Verdict {
Done,
Rejected(StatusCode, String),
Retry(String),
}
impl WriteQueue {
pub fn new(
dir: PathBuf,
app: Router,
token: String,
health: Arc<Health>,
gate: Arc<WriteGate>,
) -> Self {
Self {
dir,
app,
token,
health,
gate,
}
}
pub fn spawn(self) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = interval.tick() => {}
_ = self.gate.reopened() => {}
}
self.tick().await;
}
});
}
pub async fn tick(&self) -> Option<Pass> {
if self.gate.is_closed() {
self.report();
return None;
}
let pass = match self.try_lock() {
Ok(Some(_lock)) => {
self.remove_stale_tmp();
Some(self.replay().await)
}
Ok(None) => None,
Err(e) => {
tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
None
}
};
self.gate.open_when_drained(|| self.is_empty());
self.report();
pass
}
fn is_empty(&self) -> bool {
match entry_names(&self.dir) {
Ok(names) => names.is_empty(),
Err(e) => e.kind() == std::io::ErrorKind::NotFound,
}
}
fn try_lock(&self) -> std::io::Result<Option<File>> {
if !self.dir.is_dir() {
return Ok(None);
}
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(self.dir.join(LOCK_FILE))?;
Ok(file.try_lock_exclusive()?.then_some(file))
}
fn report(&self) {
let names = match entry_names(&self.dir) {
Ok(names) => names,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(e) => {
tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
return;
}
};
let oldest = names.first().and_then(|n| entry_time(n));
self.health
.queue_checked(names.len() as u64, oldest, Utc::now());
}
fn remove_stale_tmp(&self) {
let Ok(items) = std::fs::read_dir(&self.dir) else {
return;
};
let now = SystemTime::now();
for item in items.flatten() {
let path = item.path();
let stale = path.extension().is_some_and(|e| e == "tmp")
&& item
.metadata()
.and_then(|m| m.modified())
.is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
if stale && let Err(e) = std::fs::remove_file(&path) {
tracing::warn!("cannot remove {}: {e}", path.display());
}
}
}
async fn replay(&self) -> Pass {
let mut pass = Pass::default();
let names = match entry_names(&self.dir) {
Ok(names) => names,
Err(e) => {
pass.stopped = Some(format!("listing the queue: {e}"));
return pass;
}
};
let mut aliases = Aliases::load(&self.dir);
for name in &names {
let path = self.dir.join(name);
let entry = match read_entry(&path) {
Ok(Some(entry)) => aliases.apply(entry),
Ok(None) => continue,
Err(e) => {
set_aside(&path, &e);
continue;
}
};
let (verdict, reply) = self.send(&entry).await;
match verdict {
Verdict::Done => {
if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
aliases.add(client, stored, Utc::now());
if let Err(e) = aliases.save(&self.dir) {
pass.stopped = Some(format!("saving the id aliases: {e:#}"));
return pass;
}
}
remove_entry(&path);
pass.sent += 1;
}
Verdict::Rejected(status, message) => {
tracing::warn!(
"queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
entry.tool,
status.as_u16()
);
self.health
.write_rejected(&entry.tool, status.as_u16(), Utc::now());
remove_entry(&path);
pass.rejected += 1;
}
Verdict::Retry(why) => {
tracing::info!("write queue replay paused at {name}: {why}");
pass.stopped = Some(why);
return pass;
}
}
}
if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
&& aliases.prune(Utc::now())
&& let Err(e) = aliases.save(&self.dir)
{
tracing::warn!("cannot prune the write queue id aliases: {e:#}");
}
pass
}
async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
let request = match build_request(entry, &self.token) {
Ok(r) => r,
Err(e) => {
return (
Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
None,
);
}
};
let response = match self.app.clone().oneshot(request).await {
Ok(r) => r,
Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
};
let status = response.status();
let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
Ok(b) => b,
Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
};
let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
let verdict =
if status.is_success() || (status == StatusCode::NOT_FOUND && is_hard_forget(entry)) {
Verdict::Done
} else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
Verdict::Retry(format!("HTTP {status}"))
} else {
let message = reply
.as_ref()
.and_then(|v| v["error"].as_str().map(str::to_string))
.unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
Verdict::Rejected(status, message)
};
(verdict, reply)
}
}
fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
let method = Method::from_bytes(entry.method.as_bytes())?;
let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
Some(q) => format!("{}?{q}", entry.path),
None => entry.path.clone(),
};
let builder = Request::builder()
.method(method)
.uri(uri)
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(REPLAY_HEADER, "1");
Ok(match &entry.body {
Some(body) => builder
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.to_string()))?,
None => builder.body(Body::empty())?,
})
}
fn is_hard_forget(entry: &Entry) -> bool {
entry.method.eq_ignore_ascii_case("DELETE")
&& entry
.query
.as_deref()
.is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
}
fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
if entry.tool != "remember" {
return None;
}
let sent = entry.body.as_ref()?["id"].as_str()?;
let stored = reply?["id"].as_str()?;
(sent != stored).then(|| (sent.to_string(), stored.to_string()))
}
#[derive(Default, Serialize, Deserialize)]
struct Aliases(BTreeMap<String, Alias>);
#[derive(Serialize, Deserialize)]
struct Alias {
to: String,
at: DateTime<Utc>,
}
impl Aliases {
fn load(dir: &Path) -> Self {
match std::fs::read(dir.join(ALIASES_FILE)) {
Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
tracing::warn!("ignoring unreadable write queue id aliases: {e}");
Self::default()
}),
Err(_) => Self::default(),
}
}
fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
self.0.insert(
client,
Alias {
to: stored,
at: now,
},
);
}
fn apply(&self, mut entry: Entry) -> Entry {
for (client, alias) in &self.0 {
entry.path = entry.path.replace(client.as_str(), &alias.to);
entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
if let Some(body) = entry.body.as_mut() {
replace_strings(body, client, &alias.to);
}
}
entry
}
fn prune(&mut self, now: DateTime<Utc>) -> bool {
let before = self.0.len();
self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
self.0.len() != before
}
fn save(&self, dir: &Path) -> anyhow::Result<()> {
let path = dir.join(ALIASES_FILE);
if self.0.is_empty() {
return match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
_ => Ok(()),
};
}
let tmp = dir.join(format!(
"{ALIASES_FILE}.{}.tmp",
uuid::Uuid::new_v4().simple()
));
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&tmp)?;
file.write_all(&serde_json::to_vec(self)?)?;
file.sync_all()?;
std::fs::rename(&tmp, path)?;
Ok(())
}
}
fn read_entry(path: &Path) -> anyhow::Result<Option<Entry>> {
match std::fs::read(path) {
Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
fn set_aside(path: &Path, error: &anyhow::Error) {
let aside = path.with_extension("unreadable");
tracing::warn!(
"write queue entry {} is unreadable ({error:#}); moved to {}",
path.display(),
aside.display()
);
if let Err(e) = std::fs::rename(path, &aside) {
tracing::warn!("cannot move {} aside: {e}", path.display());
}
}
fn remove_entry(path: &Path) {
if let Err(e) = std::fs::remove_file(path)
&& e.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
}
}
fn replace_strings(value: &mut Value, from: &str, to: &str) {
match value {
Value::String(s) if s.contains(from) => *s = s.replace(from, to),
Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
_ => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_timestamped_uuid_json_names_are_entries() {
let id = "0b5c2f9e-5d0e-4c1a-9a57-0c7f4f1f2a10";
assert!(is_entry_name(&format!("1793505622000000000-{id}.json")));
for name in [
"projects.json".to_string(),
".lock".to_string(),
format!("1793505622000000000-{id}.json.tmp"),
format!("179350562200000000-{id}.json"),
format!("1793505622000000000-{id}.unreadable"),
"1793505622000000000-not-a-uuid.json".to_string(),
] {
assert!(!is_entry_name(&name), "{name}");
}
}
#[test]
fn an_entry_name_dates_the_call() {
let at = entry_time("1793505622000000000-x.json").unwrap();
assert_eq!(at.timestamp(), 1_793_505_622);
}
#[test]
fn only_a_hard_forget_treats_404_as_done() {
let forget = |query: &str| Entry {
tool: "forget".to_string(),
method: "DELETE".to_string(),
path: "/v1/memory/x".to_string(),
query: Some(query.to_string()),
body: None,
};
assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
assert!(!is_hard_forget(&forget("hard=false")));
}
}