use std::collections::BTreeMap;
use std::fs::File;
use std::io::Write;
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use age::x25519::{Identity, Recipient};
use axum::Router;
use axum::body::Body;
use axum::http::{Method, Request, StatusCode, header};
use chrono::{DateTime, Utc};
use fs4::fs_std::FileExt;
use second_brain_core::keys::{self, KeyStorage};
use second_brain_core::write_queue::{OpenError, entry_name, is_sealed_name, open};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tower::ServiceExt;
use crate::health::Health;
use crate::write_gate::{REPLAY_HEADER, WriteGate};
pub use second_brain_core::write_queue::{DIR_NAME, Entry, is_entry_name};
pub const INTERVAL: Duration = Duration::from_secs(30);
const LOCK_FILE: &str = ".lock";
const STALE_TMP: Duration = Duration::from_secs(60 * 60);
const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
const ALIASES_FILE: &str = "aliases.json";
const ALIAS_KEEP: chrono::Duration = chrono::Duration::hours(1);
const MAX_ENTRIES: usize = 10_000;
fn entry_time(name: &str) -> Option<DateTime<Utc>> {
let nanos: i64 = name.get(..19)?.parse().ok()?;
Some(DateTime::from_timestamp_nanos(nanos))
}
pub fn entry_names(dir: &Path) -> std::io::Result<Vec<String>> {
let mut names = Vec::new();
for item in std::fs::read_dir(dir)? {
let name = item?.file_name().to_string_lossy().into_owned();
if is_entry_name(&name) {
names.push(name);
}
}
names.sort();
Ok(names)
}
pub fn publish_device_identity(data_dir: &Path, storage: &KeyStorage) -> Option<Identity> {
let keys = match keys::load_device_keys_in(data_dir, storage) {
Ok(Some(keys)) => keys,
Ok(None) => {
tracing::warn!("no device key: queued writes are not sealed");
return None;
}
Err(e) => {
tracing::error!("device keys unreadable: queued writes are not sealed: {e:#}");
return None;
}
};
if let Err(e) = keys::publish_device_pub(data_dir, &keys) {
tracing::warn!("the MCP server cannot seal the writes it queues: {e:#}");
}
Some(keys.age_identity().clone())
}
pub fn push(dir: &Path, entry: &Entry, recipient: Option<&Recipient>) -> anyhow::Result<()> {
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(dir)?;
match entry_names(dir) {
Ok(names) if names.len() >= MAX_ENTRIES => {
anyhow::bail!("the write queue is full ({MAX_ENTRIES} entries)")
}
Ok(_) => {}
Err(e) => return Err(e.into()),
}
let nanos = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)?
.as_nanos();
let bytes = match recipient {
Some(recipient) => entry.to_sealed_file(std::slice::from_ref(recipient))?,
None => entry.to_plain_file()?,
};
let name = entry_name(nanos, recipient.is_some());
let tmp = dir.join(format!("{name}.tmp"));
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&tmp)?;
file.write_all(&bytes)?;
file.sync_all()?;
std::fs::rename(&tmp, dir.join(name))?;
Ok(())
}
#[derive(Debug, Default, PartialEq)]
pub struct Pass {
pub sent: usize,
pub rejected: usize,
pub waiting: usize,
pub stopped: Option<String>,
}
pub struct WriteQueue {
dir: PathBuf,
app: Router,
token: String,
health: Arc<Health>,
gate: Arc<WriteGate>,
identity: Option<Identity>,
warned_no_key: AtomicBool,
}
enum Verdict {
Done,
Rejected(StatusCode, String),
Retry(String),
}
impl WriteQueue {
pub fn new(
dir: PathBuf,
app: Router,
token: String,
health: Arc<Health>,
gate: Arc<WriteGate>,
) -> Self {
Self {
dir,
app,
token,
health,
gate,
identity: None,
warned_no_key: AtomicBool::new(false),
}
}
pub fn opening_with(mut self, identity: Identity) -> Self {
self.identity = Some(identity);
self
}
pub fn spawn(self) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = interval.tick() => {}
_ = self.gate.reopened() => {}
}
self.tick().await;
}
});
}
pub async fn tick(&self) -> Option<Pass> {
if self.gate.is_closed() {
self.report();
return None;
}
let pass = match self.try_lock() {
Ok(Some(_lock)) => {
self.remove_stale_tmp();
Some(self.replay().await)
}
Ok(None) => None,
Err(e) => {
tracing::warn!("cannot lock the write queue at {}: {e}", self.dir.display());
None
}
};
self.gate.open_when_drained(|| self.nothing_to_replay());
self.report();
pass
}
fn nothing_to_replay(&self) -> bool {
match entry_names(&self.dir) {
Ok(names) => names.iter().all(|name| self.waits_for_key(name)),
Err(e) => e.kind() == std::io::ErrorKind::NotFound,
}
}
fn waits_for_key(&self, name: &str) -> bool {
self.identity.is_none() && is_sealed_name(name)
}
fn try_lock(&self) -> std::io::Result<Option<File>> {
if !self.dir.is_dir() {
return Ok(None);
}
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(self.dir.join(LOCK_FILE))?;
Ok(file.try_lock_exclusive()?.then_some(file))
}
fn report(&self) {
let names = match entry_names(&self.dir) {
Ok(names) => names,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(e) => {
tracing::warn!("cannot list the write queue at {}: {e}", self.dir.display());
return;
}
};
let oldest = names.first().and_then(|n| entry_time(n));
self.health
.queue_checked(names.len() as u64, oldest, Utc::now());
}
fn remove_stale_tmp(&self) {
let Ok(items) = std::fs::read_dir(&self.dir) else {
return;
};
let now = SystemTime::now();
for item in items.flatten() {
let path = item.path();
let stale = path.extension().is_some_and(|e| e == "tmp")
&& item
.metadata()
.and_then(|m| m.modified())
.is_ok_and(|t| now.duration_since(t).unwrap_or_default() >= STALE_TMP);
if stale && let Err(e) = std::fs::remove_file(&path) {
tracing::warn!("cannot remove {}: {e}", path.display());
}
}
}
async fn replay(&self) -> Pass {
let mut pass = Pass::default();
let names = match entry_names(&self.dir) {
Ok(names) => names,
Err(e) => {
pass.stopped = Some(format!("listing the queue: {e}"));
return pass;
}
};
let mut aliases = Aliases::load(&self.dir);
for name in &names {
let path = self.dir.join(name);
let entry = match read_entry(&path, self.identity.as_ref()) {
Ok(Some(entry)) => aliases.apply(entry),
Ok(None) => continue,
Err(OpenError::NoKey) => {
pass.waiting += 1;
continue;
}
Err(OpenError::Unreadable(e)) => {
set_aside(&path, &e);
continue;
}
};
let (verdict, reply) = self.send(&entry).await;
match verdict {
Verdict::Done => {
if let Some((client, stored)) = remember_alias(&entry, reply.as_ref()) {
aliases.add(client, stored, Utc::now());
if let Err(e) = aliases.save(&self.dir) {
pass.stopped = Some(format!("saving the id aliases: {e:#}"));
return pass;
}
}
remove_entry(&path);
pass.sent += 1;
}
Verdict::Rejected(status, message) => {
tracing::warn!(
"queued {} rejected by the daemon (HTTP {}): {message}; dropped {name}",
entry.tool,
status.as_u16()
);
self.health
.write_rejected(&entry.tool, status.as_u16(), Utc::now());
remove_entry(&path);
pass.rejected += 1;
}
Verdict::Retry(why) => {
tracing::info!("write queue replay paused at {name}: {why}");
pass.stopped = Some(why);
return pass;
}
}
}
if entry_names(&self.dir).is_ok_and(|left| left.is_empty())
&& aliases.prune(Utc::now())
&& let Err(e) = aliases.save(&self.dir)
{
tracing::warn!("cannot prune the write queue id aliases: {e:#}");
}
if pass.waiting > 0 && !self.warned_no_key.swap(true, Ordering::Relaxed) {
tracing::warn!(
"{} sealed queued writes wait in {} for the device key, which this daemon \
could not read at start; later writes replay without them",
pass.waiting,
self.dir.display()
);
}
pass
}
async fn send(&self, entry: &Entry) -> (Verdict, Option<Value>) {
let request = match build_request(entry, &self.token) {
Ok(r) => r,
Err(e) => {
return (
Verdict::Rejected(StatusCode::BAD_REQUEST, format!("unusable entry: {e}")),
None,
);
}
};
let response = match self.app.clone().oneshot(request).await {
Ok(r) => r,
Err(e) => return (Verdict::Retry(format!("dispatch failed: {e}")), None),
};
let status = response.status();
let bytes = match axum::body::to_bytes(response.into_body(), MAX_RESPONSE_BYTES).await {
Ok(b) => b,
Err(e) => return (Verdict::Retry(format!("reading the reply: {e}")), None),
};
let reply: Option<Value> = serde_json::from_slice(&bytes).ok();
let verdict = if status.is_success()
|| (status == StatusCode::NOT_FOUND && is_idempotent_removal(entry))
{
Verdict::Done
} else if matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error() {
Verdict::Retry(format!("HTTP {status}"))
} else {
let message = reply
.as_ref()
.and_then(|v| v["error"].as_str().map(str::to_string))
.unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned());
Verdict::Rejected(status, message)
};
(verdict, reply)
}
}
fn build_request(entry: &Entry, token: &str) -> anyhow::Result<Request<Body>> {
let method = Method::from_bytes(entry.method.as_bytes())?;
let uri = match entry.query.as_deref().filter(|q| !q.is_empty()) {
Some(q) => format!("{}?{q}", entry.path),
None => entry.path.clone(),
};
let builder = Request::builder()
.method(method)
.uri(uri)
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(REPLAY_HEADER, "1");
Ok(match &entry.body {
Some(body) => builder
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.to_string()))?,
None => builder.body(Body::empty())?,
})
}
fn is_idempotent_removal(entry: &Entry) -> bool {
is_hard_forget(entry) || is_delete(entry)
}
fn is_delete(entry: &Entry) -> bool {
entry.method.eq_ignore_ascii_case("POST") && entry.path == "/v1/delete"
}
fn is_hard_forget(entry: &Entry) -> bool {
entry.method.eq_ignore_ascii_case("DELETE")
&& entry
.query
.as_deref()
.is_some_and(|q| q.split('&').any(|pair| pair == "hard=true"))
}
fn remember_alias(entry: &Entry, reply: Option<&Value>) -> Option<(String, String)> {
if entry.tool != "remember" {
return None;
}
let sent = entry.body.as_ref()?["id"].as_str()?;
let stored = reply?["id"].as_str()?;
(sent != stored).then(|| (sent.to_string(), stored.to_string()))
}
#[derive(Default, Serialize, Deserialize)]
struct Aliases(BTreeMap<String, Alias>);
#[derive(Serialize, Deserialize)]
struct Alias {
to: String,
at: DateTime<Utc>,
}
impl Aliases {
fn load(dir: &Path) -> Self {
match std::fs::read(dir.join(ALIASES_FILE)) {
Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
tracing::warn!("ignoring unreadable write queue id aliases: {e}");
Self::default()
}),
Err(_) => Self::default(),
}
}
fn add(&mut self, client: String, stored: String, now: DateTime<Utc>) {
self.0.insert(
client,
Alias {
to: stored,
at: now,
},
);
}
fn apply(&self, mut entry: Entry) -> Entry {
for (client, alias) in &self.0 {
entry.path = entry.path.replace(client.as_str(), &alias.to);
entry.query = entry.query.map(|q| q.replace(client.as_str(), &alias.to));
if let Some(body) = entry.body.as_mut() {
replace_strings(body, client, &alias.to);
}
}
entry
}
fn prune(&mut self, now: DateTime<Utc>) -> bool {
let before = self.0.len();
self.0.retain(|_, alias| now - alias.at < ALIAS_KEEP);
self.0.len() != before
}
fn save(&self, dir: &Path) -> anyhow::Result<()> {
let path = dir.join(ALIASES_FILE);
if self.0.is_empty() {
return match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
_ => Ok(()),
};
}
let tmp = dir.join(format!(
"{ALIASES_FILE}.{}.tmp",
uuid::Uuid::new_v4().simple()
));
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&tmp)?;
file.write_all(&serde_json::to_vec(self)?)?;
file.sync_all()?;
std::fs::rename(&tmp, path)?;
Ok(())
}
}
fn read_entry(path: &Path, identity: Option<&Identity>) -> Result<Option<Entry>, OpenError> {
match std::fs::read(path) {
Ok(bytes) => open(&bytes, identity).map(Some),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(OpenError::Unreadable(e.into())),
}
}
fn set_aside(path: &Path, error: &anyhow::Error) {
let aside = path.with_extension("unreadable");
tracing::warn!(
"write queue entry {} is unreadable ({error:#}); moved to {}",
path.display(),
aside.display()
);
if let Err(e) = std::fs::rename(path, &aside) {
tracing::warn!("cannot move {} aside: {e}", path.display());
}
}
fn remove_entry(path: &Path) {
if let Err(e) = std::fs::remove_file(path)
&& e.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!("cannot remove replayed entry {}: {e}", path.display());
}
}
fn replace_strings(value: &mut Value, from: &str, to: &str) {
match value {
Value::String(s) if s.contains(from) => *s = s.replace(from, to),
Value::Array(items) => items.iter_mut().for_each(|v| replace_strings(v, from, to)),
Value::Object(map) => map.values_mut().for_each(|v| replace_strings(v, from, to)),
_ => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_device_identity_is_published_for_the_mcp_server() {
let dir = tempfile::tempdir().unwrap();
assert!(publish_device_identity(dir.path(), &KeyStorage::File).is_none());
assert!(!dir.path().join(keys::DEVICE_PUB_FILE).exists());
let device = keys::create_device_keys(dir.path(), &KeyStorage::File).unwrap();
let identity = publish_device_identity(dir.path(), &KeyStorage::File).unwrap();
assert_eq!(identity.to_public(), device.age_recipient());
let published = std::fs::read_to_string(dir.path().join(keys::DEVICE_PUB_FILE)).unwrap();
assert_eq!(published.trim(), device.age_recipient().to_string());
}
#[test]
fn an_entry_name_dates_the_call() {
let at = entry_time("1793505622000000000-x.json").unwrap();
assert_eq!(at.timestamp(), 1_793_505_622);
}
#[test]
fn only_a_hard_forget_treats_404_as_done() {
let forget = |query: &str| Entry {
tool: "forget".to_string(),
method: "DELETE".to_string(),
path: "/v1/memory/x".to_string(),
query: Some(query.to_string()),
body: None,
};
assert!(is_hard_forget(&forget("hard=true&reason=wrong")));
assert!(!is_hard_forget(&forget("hard=false")));
assert!(is_idempotent_removal(&forget("hard=true")));
assert!(!is_idempotent_removal(&forget("hard=false")));
}
#[test]
fn a_replayed_delete_treats_404_as_done_and_other_posts_do_not() {
let post = |path: &str| Entry {
tool: "delete".to_string(),
method: "POST".to_string(),
path: path.to_string(),
query: None,
body: Some(serde_json::json!({"session": "s1"})),
};
assert!(is_idempotent_removal(&post("/v1/delete")));
assert!(!is_idempotent_removal(&post("/v1/remember")));
}
}