mod common;
use std::io::{BufReader, Read, Write};
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use anyhow::Result;
use axum::http::{HeaderValue, Request, StatusCode, header};
use chrono::{SecondsFormat, Utc};
use http_body_util::BodyExt;
use serde_json::Value;
use tower::ServiceExt;
use common::{app_in, get, json_body, post_json, state_in, upgrade_from_sync_key};
use second_brain_api::clock_gate::{ClockGate, READING_FILE, TimeSignals};
use second_brain_api::health::Health;
use second_brain_api::sync_loop::{Connect, SyncLoop};
use second_brain_api::write_gate::WriteGate;
use second_brain_core::schema::{Memory, MemoryType, Scope};
use second_brain_core::sqlite_store::{SqliteStore, Truncation};
use second_brain_core::store::{Store, StoreAdmin};
use second_brain_sync::crypto::seal_line;
use second_brain_sync::dial::Route;
use second_brain_sync::export::export_changes;
use second_brain_sync::keyring::{IdentityMismatch, Keyring, LEGACY_KEY_FILE};
use second_brain_sync::peers;
use second_brain_sync::protocol::{BelowFloorLine, Hello, RefusedLine, SyncHeader};
use second_brain_sync::session::{
self, Exit, Exported, Imported, Local, PeerRefused, Remote, Transport, read_checked_header,
};
const TOKEN: &str = "sync-loop-test-token";
const EXPORT_PAGE: usize = 2000;
const FAR_FUTURE_BUILD_SECS: i64 = 4_102_444_800;
const SHARED_SYNC_KEY: [u8; 32] = [7; 32];
fn authorized(mut request: Request<axum::body::Body>) -> Request<axum::body::Body> {
request.headers_mut().insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {TOKEN}")).unwrap(),
);
request
}
#[derive(Clone)]
struct PeerApi {
app: axum::Router,
runtime: tokio::runtime::Handle,
}
impl PeerApi {
fn send(&self, request: Request<axum::body::Body>) -> Result<(StatusCode, bytes::Bytes)> {
let request = authorized(request);
let app = self.app.clone();
self.runtime.block_on(async move {
let response = app
.oneshot(request)
.await
.unwrap_or_else(|never| match never {});
let status = response.status();
Ok((status, response.into_body().collect().await?.to_bytes()))
})
}
fn post(&self, uri: &str, body: Value) -> Result<Value> {
let request = authorized(post_json(uri, body));
let app = self.app.clone();
self.runtime.block_on(async move {
let response = app
.oneshot(request)
.await
.unwrap_or_else(|never| match never {});
let status = response.status();
let bytes = response.into_body().collect().await?.to_bytes();
if !status.is_success() {
anyhow::bail!(
"{uri} answered {status}: {}",
String::from_utf8_lossy(&bytes)
);
}
if bytes.is_empty() {
return Ok(Value::Null);
}
Ok(serde_json::from_slice(&bytes)?)
})
}
}
struct InProcessPeer {
host: String,
machine_id: String,
api: PeerApi,
}
impl Transport for InProcessPeer {
fn host(&self) -> &str {
&self.host
}
fn whoami(&self) -> Result<String> {
Ok(self.machine_id.clone())
}
fn export(
&self,
after_seq: u64,
hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
let (output, mut writer) = std::io::pipe()?;
let api = self.api.clone();
let hello = hello.clone();
let worker = thread::spawn(move || -> Result<()> {
let mut cursor = after_seq;
let mut header_written = false;
loop {
let (status, body) = api.send(post_json(
"/v1/sync/export",
serde_json::json!({
"after_seq": cursor,
"limit": EXPORT_PAGE,
"header": SyncHeader::local(),
"hello": hello,
}),
))?;
let page: Value = serde_json::from_slice(&body)?;
if status == StatusCode::CONFLICT && page["below_floor"] == true {
let below = BelowFloorLine {
after_seq: page["after_seq"].as_u64().unwrap_or(cursor),
log_floor: page["log_floor"].as_u64().unwrap_or(0),
};
writeln!(writer, "{}", below.to_line())?;
return Ok(());
}
if status == StatusCode::CONFLICT
&& let Some(refused) =
RefusedLine::from_refusal_body(&String::from_utf8_lossy(&body))
{
if !header_written {
writeln!(writer, "{}", refused.to_line())?;
}
anyhow::bail!("export answered {status}: {}", refused.error);
}
anyhow::ensure!(status.is_success(), "export answered {status}: {page}");
if !header_written {
let header = SyncHeader {
hello: serde_json::from_value(page["hello"].clone())?,
..SyncHeader::local()
};
writeln!(writer, "{}", header.to_line())?;
header_written = true;
}
let lines: Vec<String> = serde_json::from_value(page["lines"].clone())?;
for line in &lines {
writeln!(writer, "{line}")?;
}
cursor = page["max_seq"].as_u64().unwrap_or(cursor);
if lines.len() < EXPORT_PAGE {
return Ok(());
}
}
});
Ok((Box::new(output), Box::new(PipedRemote(Some(worker)))))
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
let (reader, input) = std::io::pipe()?;
let (replies, mut output) = std::io::pipe()?;
let api = self.api.clone();
let worker = thread::spawn(move || -> Result<()> {
let mut reader = BufReader::new(reader);
let header = read_checked_header(&mut reader, "the pushing daemon")?;
session::serve_push(&api, &header, reader, &mut output)?;
Ok(())
});
Ok((
Box::new(input),
Box::new(replies),
Box::new(PipedRemote(Some(worker))),
))
}
fn snapshot_export(&self, hello: &Hello) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
let (output, mut writer) = std::io::pipe()?;
let api = self.api.clone();
let hello = hello.clone();
let worker = thread::spawn(move || -> Result<()> {
let (status, body) = api.send(post_json(
"/v1/sync/snapshot",
serde_json::json!({"hello": hello, "with_chunks": true}),
))?;
anyhow::ensure!(
status.is_success(),
"snapshot answered {status}: {}",
String::from_utf8_lossy(&body)
);
writer.write_all(&body)?;
Ok(())
});
Ok((Box::new(output), Box::new(PipedRemote(Some(worker)))))
}
fn snapshot_import(
&self,
) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
let (mut reader, input) = std::io::pipe()?;
let (replies, mut output) = std::io::pipe()?;
let api = self.api.clone();
let worker = thread::spawn(move || -> Result<()> {
let mut stream = Vec::new();
reader.read_to_end(&mut stream)?;
let request = Request::post("/v1/sync/snapshot/apply")
.header(header::CONTENT_TYPE, "application/octet-stream")
.body(axum::body::Body::from(stream))?;
let (status, body) = api.send(request)?;
anyhow::ensure!(
status.is_success(),
"snapshot apply answered {status}: {}",
String::from_utf8_lossy(&body)
);
writeln!(output, "{}", String::from_utf8_lossy(&body))?;
Ok(())
});
Ok((
Box::new(input),
Box::new(replies),
Box::new(PipedRemote(Some(worker))),
))
}
}
impl Local for PeerApi {
fn cursors(&self) -> Result<Vec<(String, u64)>> {
let (status, body) = self.send(get("/v1/sync/status"))?;
anyhow::ensure!(status.is_success(), "sync/status answered {status}");
let answer: Value = serde_json::from_slice(&body)?;
Ok(answer["peers"]
.as_array()
.into_iter()
.flatten()
.filter_map(|p| Some((p["peer_id"].as_str()?.to_string(), p["last_seq"].as_u64()?)))
.collect())
}
fn set_cursor(&self, _peer_id: &str, _last_seq: u64) -> Result<()> {
unreachable!("serving a push sets no cursor")
}
fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported> {
let resp = self.post(
"/v1/sync/import",
serde_json::json!({"header": header, "lines": lines}),
)?;
let count = |field: &str| resp[field].as_u64().unwrap_or(0);
Ok(Imported {
created: count("created"),
updated: count("updated"),
deleted: count("deleted"),
skipped: count("skipped"),
errors: count("errors"),
max_seq: count("max_seq"),
reached: count("reached"),
..Imported::default()
})
}
fn export(&self, _peer_id: &str, _after_seq: u64, _limit: usize) -> Result<Exported> {
unreachable!("serving a push exports nothing")
}
fn hello_for(&self, peer_id: &str) -> Result<Hello> {
Ok(serde_json::from_value(self.post(
"/v1/sync/hello",
serde_json::json!({"peer_id": peer_id}),
)?)?)
}
fn accept_hello(&self, hello: &Hello) -> Result<()> {
let (status, body) = self.send(post_json(
"/v1/sync/accept-hello",
serde_json::json!({"hello": hello}),
))?;
let body = String::from_utf8_lossy(&body);
if status == StatusCode::CONFLICT
&& let Some(refused) = RefusedLine::from_refusal_body(&body).filter(|r| r.identity)
{
return Err(IdentityMismatch(refused.error).into());
}
anyhow::ensure!(
status.is_success(),
"/v1/sync/accept-hello answered {status}: {body}"
);
Ok(())
}
fn record_ack(&self, _peer_id: &str, _acked_seq: u64) -> Result<()> {
unreachable!("serving a push records no ack")
}
fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
self.post(
"/v1/sync/peer-in",
serde_json::json!({"peer_id": peer_id, "seq": seq}),
)?;
Ok(())
}
fn push_report(&self, _peer_id: &str) -> Result<Option<u64>> {
unreachable!("serving a push takes no report")
}
fn record_push_report(&self, _peer_id: &str, _reported: Option<u64>) -> Result<()> {
unreachable!("serving a push takes no report")
}
}
struct PipedRemote(Option<JoinHandle<Result<()>>>);
impl Remote for PipedRemote {
fn wait(&mut self) -> Result<Exit> {
let Some(worker) = self.0.take() else {
anyhow::bail!("remote already waited on");
};
Ok(match worker.join() {
Ok(Ok(())) => Exit {
success: true,
status: "exit status: 0".to_string(),
},
Ok(Err(e)) => Exit {
success: false,
status: format!("{e:#}"),
},
Err(_) => Exit {
success: false,
status: "panicked".to_string(),
},
})
}
fn stop(&mut self) -> Result<()> {
self.wait().map(|_| ())
}
}
struct DownPeer {
host: String,
calls: Arc<AtomicUsize>,
}
impl Transport for DownPeer {
fn host(&self) -> &str {
&self.host
}
fn whoami(&self) -> Result<String> {
self.calls.fetch_add(1, Ordering::SeqCst);
anyhow::bail!("ssh: connect to host down: Connection refused")
}
fn export(
&self,
_after_seq: u64,
_hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
anyhow::bail!("a down peer exports nothing")
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
anyhow::bail!("a down peer imports nothing")
}
}
struct HungPeer {
host: String,
}
impl Transport for HungPeer {
fn host(&self) -> &str {
&self.host
}
fn whoami(&self) -> Result<String> {
thread::sleep(Duration::from_secs(60));
anyhow::bail!("ssh to a hung peer timed out")
}
fn export(
&self,
_after_seq: u64,
_hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
anyhow::bail!("a hung peer exports nothing")
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
anyhow::bail!("a hung peer imports nothing")
}
}
async fn wait_until(within: Duration, mut done: impl FnMut() -> bool) -> bool {
let deadline = Instant::now() + within;
while Instant::now() < deadline {
if done() {
return true;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
done()
}
fn synced(data_dir: &Path) -> bool {
peers::last_success(data_dir).unwrap().is_some()
}
async fn recalls(app: &axum::Router, token: &str) -> bool {
let response = app
.clone()
.oneshot(post_json("/v1/recall", serde_json::json!({"query": token})))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
json_body(response).await["memories"]
.as_array()
.unwrap()
.iter()
.any(|m| m["content"].as_str().is_some_and(|c| c.contains(token)))
}
struct Daemons {
app_a: axum::Router,
app_b: axum::Router,
data_dir: tempfile::TempDir,
data_dir_b: tempfile::TempDir,
sqlite_a: Arc<SqliteStore>,
sqlite_b: Arc<SqliteStore>,
down_calls: Arc<AtomicUsize>,
}
fn upgraded(
machine_id: &str,
dir: &Path,
key: [u8; 32],
knows: &[&str],
) -> (axum::Router, Arc<SqliteStore>) {
let store = SqliteStore::in_memory(machine_id.to_string()).unwrap();
upgrade_from_sync_key(&store, dir, key, knows);
app_in(store, dir)
}
fn fresh_health() -> Arc<Health> {
Arc::new(Health::new(Utc::now(), None))
}
fn spawn_daemons(peer_lines: &str) -> Daemons {
spawn_daemons_with_keys(peer_lines, SHARED_SYNC_KEY, SHARED_SYNC_KEY)
}
fn spawn_daemons_with_keys(peer_lines: &str, key_a: [u8; 32], key_b: [u8; 32]) -> Daemons {
let (daemons, sync_loop) = daemons_with_keys(peer_lines, key_a, key_b);
sync_loop.spawn();
daemons
}
fn daemons_with_keys(peer_lines: &str, key_a: [u8; 32], key_b: [u8; 32]) -> (Daemons, SyncLoop) {
let data_dir = tempfile::tempdir().unwrap();
let data_dir_b = tempfile::tempdir().unwrap();
let (app_a, sqlite_a) = upgraded("machine-a", data_dir.path(), key_a, &["machine-b"]);
let (app_b, sqlite_b) = upgraded("machine-b", data_dir_b.path(), key_b, &["machine-a"]);
let api_b = PeerApi {
app: app_b.clone(),
runtime: tokio::runtime::Handle::current(),
};
std::fs::write(data_dir.path().join(peers::PEERS_FILE), peer_lines).unwrap();
let down_calls = Arc::new(AtomicUsize::new(0));
let connect: Connect = {
let down_calls = down_calls.clone();
Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
let Route::Line(peer) = route else {
anyhow::bail!("no device routes here");
};
let host = peer.host.clone();
Ok(match peer.host.as_str() {
"user@b" => Box::new(InProcessPeer {
host,
machine_id: "machine-b".to_string(),
api: api_b.clone(),
}),
"user@down" => Box::new(DownPeer {
host,
calls: down_calls.clone(),
}),
"user@hung" => Box::new(HungPeer { host }),
other => anyhow::bail!("no such peer {other}"),
})
})
};
let sync_loop = SyncLoop {
data_dir: data_dir.path().to_path_buf(),
sqlite: sqlite_a.clone(),
gate: Arc::new(WriteGate::default()),
clock: ClockGate::open_for_tests(),
health: fresh_health(),
access: Arc::default(),
bootstrap: Arc::default(),
connect,
discovery: None,
};
(
Daemons {
app_a,
app_b,
data_dir,
data_dir_b,
sqlite_a,
sqlite_b,
down_calls,
},
sync_loop,
)
}
#[tokio::test(flavor = "multi_thread")]
async fn a_memory_remembered_on_one_daemon_is_recallable_on_its_peer_within_ten_seconds() {
let Daemons {
app_a,
app_b,
data_dir,
data_dir_b,
sqlite_a,
sqlite_b,
down_calls,
} = spawn_daemons("user@b\nuser@down\nuser@hung\n");
for dir in [&data_dir, &data_dir_b] {
assert!(
!dir.path().join(LEGACY_KEY_FILE).exists(),
"the upgrade deletes sync.key"
);
}
assert!(
wait_until(Duration::from_secs(10), || synced(data_dir.path())).await,
"the start-up full sync with b never succeeded while the hung peer blocked"
);
assert!(
wait_until(Duration::from_secs(10), || down_calls
.load(Ordering::SeqCst)
> 0)
.await,
"the down peer was never tried at start-up"
);
let token = format!("synctoken{}", uuid::Uuid::new_v4().simple());
let response = app_a
.clone()
.oneshot(post_json(
"/v1/remember",
serde_json::json!({"content": format!("the sync loop carried {token} across")}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let remembered = Instant::now();
let mut found = false;
while remembered.elapsed() < Duration::from_secs(10) {
if recalls(&app_b, &token).await {
found = true;
break;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
assert!(
found,
"b could not recall the memory 10 s after a remembered it"
);
let mut epochs_a = Keyring::load(&sqlite_a, data_dir.path())
.unwrap()
.held_epochs();
let mut epochs_b = Keyring::load(&sqlite_b, data_dir_b.path())
.unwrap()
.held_epochs();
epochs_a.sort_by_key(|e| (e.number, e.key_id));
epochs_b.sort_by_key(|e| (e.number, e.key_id));
assert_eq!(epochs_a.len(), 3, "epoch 0 and each daemon's epoch 1");
assert_eq!(epochs_a, epochs_b);
tokio::time::sleep(Duration::from_secs(2)).await;
assert_eq!(
down_calls.load(Ordering::SeqCst),
1,
"the write-triggered push retried the down peer inside its backoff"
);
assert!(synced(data_dir.path()));
}
async fn sibling_note(app: &axum::Router, project: &str, since: &str, id: &str) -> Option<Value> {
let response = app
.clone()
.oneshot(get(&format!("/v1/notes?project={project}&since={since}")))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
json_body(response).await["notes"]
.as_array()
.unwrap()
.iter()
.find(|n| n["id"] == id)
.cloned()
}
#[tokio::test(flavor = "multi_thread")]
async fn a_note_written_on_one_daemon_is_in_its_peers_since_lookup_within_ten_seconds() {
let daemons = spawn_daemons("user@b\n");
assert!(
wait_until(Duration::from_secs(10), || synced(daemons.data_dir.path())).await,
"the start-up full sync with b never succeeded"
);
let response = daemons
.app_a
.clone()
.oneshot(post_json(
"/v1/notes",
serde_json::json!({
"session_id": "session-a",
"project": "-p-sib",
"kind": "fact",
"decision": "the sibling port is 7311",
}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let id = json_body(response).await["id"]
.as_str()
.unwrap()
.to_string();
let written = Instant::now();
let since =
(Utc::now() - chrono::Duration::hours(2)).to_rfc3339_opts(SecondsFormat::Secs, true);
let mut found = None;
while written.elapsed() < Duration::from_secs(10) {
found = sibling_note(&daemons.app_b, "-p-sib", &since, &id).await;
if found.is_some() {
break;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
let found = found.expect("b's since lookup lacked a's note 10 s after the write");
assert_eq!(found["session_id"], "session-a");
assert_eq!(found["kind"], "fact");
assert_eq!(found["decision"], "the sibling port is 7311");
}
struct FakeTime {
synced: bool,
}
impl TimeSignals for FakeTime {
fn now_micros(&self) -> i64 {
Utc::now().timestamp_micros()
}
fn ntp_synced(&self) -> bool {
self.synced
}
fn rtc_secs(&self) -> Option<i64> {
None
}
}
fn gated_clock(dir: &Path, synced: bool) -> Arc<ClockGate> {
let hour_ago = Utc::now().timestamp_micros() - 3_600_000_000;
std::fs::write(dir.join(READING_FILE), hour_ago.to_string()).unwrap();
ClockGate::new(dir, Arc::new(FakeTime { synced }), FAR_FUTURE_BUILD_SECS)
}
fn gated_daemon(
machine_id: &str,
clock: &Arc<ClockGate>,
data_dir: &Path,
knows: &[&str],
) -> (axum::Router, Arc<SqliteStore>) {
let store = SqliteStore::in_memory(machine_id.to_string())
.unwrap()
.with_wall_clock(clock.clone());
upgrade_from_sync_key(&store, data_dir, SHARED_SYNC_KEY, knows);
let state = state_in(store, data_dir, clock.clone());
let sqlite = state.sqlite.clone();
(second_brain_api::app(state, TOKEN.to_string()), sqlite)
}
fn remember(content: &str) -> Request<axum::body::Body> {
authorized(post_json(
"/v1/remember",
serde_json::json!({"content": content}),
))
}
#[tokio::test(flavor = "multi_thread")]
async fn an_untrusted_clock_holds_writes_until_a_synced_peer_handshakes() {
let clock_dir_b = tempfile::tempdir().unwrap();
let clock_b = gated_clock(clock_dir_b.path(), false);
let (app_b, sqlite_b) = gated_daemon("machine-b", &clock_b, clock_dir_b.path(), &["machine-a"]);
let held = app_b
.clone()
.oneshot(remember("written while the clock is untrusted"))
.await
.unwrap();
assert_eq!(held.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
held.headers()
.get(header::RETRY_AFTER)
.and_then(|v| v.to_str().ok()),
Some("30")
);
assert!(!clock_b.is_open());
let clock_dir_a = tempfile::tempdir().unwrap();
let clock_a = gated_clock(clock_dir_a.path(), true);
let data_dir = tempfile::tempdir().unwrap();
let sqlite_a = SqliteStore::in_memory("machine-a".into()).unwrap();
upgrade_from_sync_key(&sqlite_a, data_dir.path(), SHARED_SYNC_KEY, &["machine-b"]);
let sqlite_a = Arc::new(sqlite_a);
let mut carried = Memory::new(
"written on the synced daemon".to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
carried.machine_id = "machine-a".to_string();
sqlite_a.store_memory(&carried).unwrap();
std::fs::write(data_dir.path().join(peers::PEERS_FILE), "user@b\n").unwrap();
let api_b = PeerApi {
app: app_b.clone(),
runtime: tokio::runtime::Handle::current(),
};
let connect: Connect = Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
Ok(Box::new(InProcessPeer {
host: route.describe(),
machine_id: "machine-b".to_string(),
api: api_b.clone(),
}))
});
SyncLoop {
data_dir: data_dir.path().to_path_buf(),
sqlite: sqlite_a,
gate: Arc::new(WriteGate::default()),
clock: clock_a,
health: fresh_health(),
access: Arc::default(),
bootstrap: Arc::default(),
connect,
discovery: None,
}
.spawn();
let deadline = Instant::now() + Duration::from_secs(10);
let mut status = held.status();
while Instant::now() < deadline {
status = app_b
.clone()
.oneshot(remember("written once a synced peer vouched for the time"))
.await
.unwrap()
.status();
if status == StatusCode::OK {
break;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
assert_eq!(
status,
StatusCode::OK,
"b still held writes 10 s after a synced peer started syncing with it"
);
assert!(clock_b.is_open());
assert!(
wait_until(Duration::from_secs(10), || synced(data_dir.path())).await,
"the sync that opened b never completed"
);
assert!(
sqlite_b.get_memory(carried.id).unwrap().is_some(),
"the push after the handshake did not reach b"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_hello_a_day_ahead_of_a_synced_daemon_is_refused_and_shows_in_health() {
let clock_dir = tempfile::tempdir().unwrap();
let clock = gated_clock(clock_dir.path(), true);
let (app, sqlite) = gated_daemon("machine-r", &clock, clock_dir.path(), &[]);
let before = sqlite.clock_state().unwrap();
let ahead = Hello {
machine_id: "far-ahead-peer".to_string(),
wall_micros: Utc::now().timestamp_micros() + 2 * 86_400_000_000,
synced: true,
cursor: None,
held_origin_seq: None,
keys: Some(
Keyring::load(&sqlite, clock_dir.path())
.unwrap()
.handshake_for("far-ahead-peer", None)
.unwrap(),
),
peer_in: None,
log_floor: 0,
head: 0,
applied: None,
merging: None,
};
let response = app
.clone()
.oneshot(authorized(post_json(
"/v1/sync/accept-hello",
serde_json::json!({"hello": ahead}),
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
assert_eq!(
sqlite.clock_state().unwrap(),
before,
"a refused hello raised a clock floor"
);
let health = json_body(app.oneshot(authorized(get("/health"))).await.unwrap()).await;
assert_eq!(health["state"], "degraded", "{health}");
let reason = health["reason"].as_str().unwrap_or_default();
assert!(
reason.contains("from far-ahead-peer: wall clock"),
"{health}"
);
}
#[tokio::test]
async fn an_import_a_closed_gate_cannot_mint_for_asks_the_sender_to_retry() {
let clock_dir = tempfile::tempdir().unwrap();
let clock = gated_clock(clock_dir.path(), false);
let store = SqliteStore::in_memory("machine-c".to_string())
.unwrap()
.with_wall_clock(clock.clone());
upgrade_from_sync_key(&store, clock_dir.path(), SHARED_SYNC_KEY, &[]);
let state = state_in(store, clock_dir.path(), clock.clone());
let sqlite = state.sqlite.clone();
let routes_alone = second_brain_api::routes::router().with_state(state);
let peer = SqliteStore::in_memory("peer-machine".to_string()).unwrap();
let peer_dir = tempfile::tempdir().unwrap();
upgrade_from_sync_key(&peer, peer_dir.path(), SHARED_SYNC_KEY, &[]);
let mut memory = Memory::new(
"relayed to a daemon that cannot mint".to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
memory.machine_id = "peer-machine".to_string();
peer.store_memory(&memory).unwrap();
let mut exported = Vec::new();
export_changes(&peer, 0, &mut exported).unwrap();
let sealer = Keyring::load(&peer, peer_dir.path())
.unwrap()
.sealer_for("machine-c", None)
.unwrap();
let lines: Vec<String> = String::from_utf8(exported)
.unwrap()
.lines()
.map(|line| seal_line(&sealer, line.as_bytes()).unwrap())
.collect();
let response = routes_alone
.oneshot(post_json(
"/v1/sync/import",
serde_json::json!({"header": SyncHeader::local(), "lines": lines}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
response
.headers()
.get(header::RETRY_AFTER)
.and_then(|v| v.to_str().ok()),
Some("30")
);
assert!(sqlite.get_memory(memory.id).unwrap().is_none());
}
#[tokio::test(flavor = "multi_thread")]
async fn daemons_upgraded_from_different_sync_keys_refuse_each_other_naming_pairing() {
let data_dir = tempfile::tempdir().unwrap();
let data_dir_b = tempfile::tempdir().unwrap();
let (app_a, _) = upgraded("machine-a", data_dir.path(), [7; 32], &["machine-b"]);
let (app_b, _) = upgraded("machine-b", data_dir_b.path(), [8; 32], &["machine-a"]);
let hello_a = json_body(
app_a
.clone()
.oneshot(authorized(post_json(
"/v1/sync/hello",
serde_json::json!({"peer_id": "machine-b"}),
)))
.await
.unwrap(),
)
.await;
for (uri, body) in [
(
"/v1/sync/export",
serde_json::json!({
"after_seq": 0,
"limit": EXPORT_PAGE,
"header": SyncHeader::local(),
"hello": hello_a,
}),
),
(
"/v1/sync/accept-hello",
serde_json::json!({"hello": hello_a}),
),
] {
let response = app_b
.clone()
.oneshot(authorized(post_json(uri, body)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT, "{uri}");
let error = json_body(response).await["error"]
.as_str()
.unwrap_or_default()
.to_string();
assert!(error.contains("different sync identity"), "{uri}: {error}");
assert!(error.contains("sb pair"), "{uri}: {error}");
assert!(!error.contains("decrypt failed"), "{uri}: {error}");
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_full_sync_and_a_push_to_a_peer_of_another_identity_fail_naming_pairing() {
let data_dir = tempfile::tempdir().unwrap();
let data_dir_b = tempfile::tempdir().unwrap();
let (app_a, _) = upgraded("machine-a", data_dir.path(), [7; 32], &["machine-b"]);
let (app_b, _) = upgraded("machine-b", data_dir_b.path(), [8; 32], &["machine-a"]);
let runtime = tokio::runtime::Handle::current();
let local_a = PeerApi {
app: app_a,
runtime: runtime.clone(),
};
let peer_b = InProcessPeer {
host: "user@b".to_string(),
machine_id: "machine-b".to_string(),
api: PeerApi {
app: app_b,
runtime,
},
};
let dir_a = data_dir.path().to_path_buf();
let jobs = tokio::task::spawn_blocking(move || {
let full = session::sync(&local_a, &peer_b, &dir_a, &|_| {}).unwrap_err();
let push = session::push(&local_a, &peer_b, "machine-b", &|_| {}).unwrap_err();
[("full", full), ("push", push)]
});
let jobs = tokio::time::timeout(Duration::from_secs(30), jobs)
.await
.expect("a refused sync hung")
.unwrap();
for (job, error) in &jobs {
let refused = error
.downcast_ref::<PeerRefused>()
.unwrap_or_else(|| panic!("{job}: not the peer's refusal: {error:#}"));
assert_eq!(refused.host, "user@b", "{job}");
assert!(refused.identity, "{job}: {error:#}");
assert!(refused.error.contains("sb pair"), "{job}: {error:#}");
assert!(
!refused.error.contains("accepting the sync hello"),
"{job}: {error:#}"
);
assert!(error.to_string().contains("sb pair"), "{job}: {error}");
}
assert!(!synced(data_dir.path()));
}
#[tokio::test(flavor = "multi_thread")]
async fn a_private_memory_never_reaches_a_device_of_another_identity() {
let data_dir = tempfile::tempdir().unwrap();
let data_dir_b = tempfile::tempdir().unwrap();
let (app_a, store_a) = upgraded("machine-a", data_dir.path(), [7; 32], &["machine-b"]);
let (app_b, _) = upgraded("machine-b", data_dir_b.path(), [8; 32], &["machine-a"]);
let content = "the shed key hangs behind the third paint tin";
let private = Memory::new(
content.to_string(),
MemoryType::Semantic,
"api".to_string(),
String::new(),
);
store_a.store_memory(&private).unwrap();
assert_eq!(
store_a.get_memory(private.id).unwrap().unwrap().scope,
Scope::Private
);
let hello_b = json_body(
app_b
.clone()
.oneshot(authorized(post_json(
"/v1/sync/hello",
serde_json::json!({"peer_id": "machine-a"}),
)))
.await
.unwrap(),
)
.await;
for (uri, body) in [
(
"/v1/sync/export",
serde_json::json!({
"after_seq": 0,
"limit": EXPORT_PAGE,
"header": SyncHeader::local(),
"hello": hello_b,
}),
),
(
"/v1/sync/snapshot",
serde_json::json!({"hello": hello_b, "with_chunks": true}),
),
] {
let response = app_a
.clone()
.oneshot(authorized(post_json(uri, body)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT, "{uri}");
let bytes = response.into_body().collect().await.unwrap().to_bytes();
let text = String::from_utf8_lossy(&bytes);
assert!(!text.contains(content), "{uri}: {text}");
assert!(!text.contains(&private.id.to_string()), "{uri}: {text}");
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_pull_from_a_daemon_before_sync_identities_is_told_to_upgrade() {
let dir = tempfile::tempdir().unwrap();
let (app, _) = upgraded("machine-b", dir.path(), SHARED_SYNC_KEY, &["machine-a"]);
let old_hello = serde_json::json!({
"machine_id": "machine-a",
"wall_micros": Utc::now().timestamp_micros(),
"synced": false,
"cursor": null,
"held_origin_seq": null,
});
let response = app
.oneshot(authorized(post_json(
"/v1/sync/export",
serde_json::json!({
"after_seq": 0,
"limit": EXPORT_PAGE,
"header": SyncHeader::local(),
"hello": old_hello,
}),
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
let error = json_body(response).await["error"]
.as_str()
.unwrap_or_default()
.to_string();
assert!(error.contains("upgrade machine-a"), "{error}");
assert!(!error.contains("sb pair"), "{error}");
}
#[tokio::test(flavor = "multi_thread")]
async fn a_full_sync_leaves_each_daemon_with_the_others_ack_and_the_receiver_with_peer_in() {
let Daemons {
app_a,
data_dir,
data_dir_b: _data_dir_b,
sqlite_a,
sqlite_b,
..
} = spawn_daemons("user@b\n");
assert!(wait_until(Duration::from_secs(10), || synced(data_dir.path())).await);
let response = app_a
.clone()
.oneshot(post_json(
"/v1/remember",
serde_json::json!({"content": "a write for the floor's bookkeeping"}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let head_a = sqlite_a.log_head().unwrap();
let pushed = wait_until(Duration::from_secs(10), || {
sqlite_b.peer_in("machine-a").unwrap() == Some(head_a)
})
.await;
assert!(pushed, "b never recorded a's push through a's head");
let ack_of = |store: &SqliteStore, peer: &str| {
store
.peer_acks()
.unwrap()
.into_iter()
.find(|ack| ack.peer_id == peer)
.and_then(|ack| ack.acked_seq)
};
assert!(
wait_until(Duration::from_secs(10), || ack_of(&sqlite_a, "machine-b")
== Some(head_a))
.await,
"a never recorded b's ack of its push"
);
assert!(
ack_of(&sqlite_b, "machine-a").is_some(),
"b never recorded a's pull as an ack"
);
assert_eq!(
sqlite_a.floor().unwrap(),
head_a,
"b has applied all of a's log"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_pull_below_the_log_floor_is_refused_as_below_floor_on_any_page() {
let dir = tempfile::tempdir().unwrap();
let (app, sqlite) = upgraded("machine-a", dir.path(), SHARED_SYNC_KEY, &["machine-b"]);
for n in 0..3 {
sqlite
.store_memory(&Memory::new(
format!("below the floor {n}"),
MemoryType::Semantic,
"test".to_string(),
String::new(),
))
.unwrap();
}
let seqs: Vec<u64> = sqlite
.sync_log_since(0)
.unwrap()
.iter()
.map(|e| e.local_seq as u64)
.collect();
let floor = seqs[seqs.len() - 2];
sqlite.record_peer_ack("machine-b", floor, 0).unwrap();
sqlite
.run_floor_pass(Truncation::On, Utc::now().timestamp_micros())
.unwrap();
assert_eq!(sqlite.log_floor().unwrap(), floor);
let export = |after_seq: u64| {
authorized(post_json(
"/v1/sync/export",
serde_json::json!({
"after_seq": after_seq,
"limit": EXPORT_PAGE,
"header": SyncHeader::local(),
"peer_id": "machine-b",
}),
))
};
for after_seq in [0, seqs[0], floor - 1] {
let response = app.clone().oneshot(export(after_seq)).await.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
let body = json_body(response).await;
assert_eq!(body["below_floor"], true, "{body}");
}
let response = app.clone().oneshot(export(floor)).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = json_body(response).await;
assert_eq!(body["lines"].as_array().map(Vec::len), Some(1), "{body}");
}
fn ids_of(store: &SqliteStore) -> std::collections::BTreeSet<uuid::Uuid> {
store.all_memory_ids().unwrap().into_iter().collect()
}
fn store_note(store: &SqliteStore, text: &str) -> Memory {
let memory = Memory::new(
text.to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
store.store_memory(&memory).unwrap();
memory
}
fn truncate_below(store: &SqliteStore, acked_by: &str, keep: usize) -> u64 {
let seqs: Vec<u64> = store
.sync_log_since(0)
.unwrap()
.iter()
.map(|e| e.local_seq as u64)
.collect();
let floor = seqs[seqs.len() - 1 - keep];
store
.record_peer_ack(acked_by, floor, Utc::now().timestamp_micros())
.unwrap();
let pass = store
.run_floor_pass(Truncation::On, Utc::now().timestamp_micros())
.unwrap();
assert!(pass.dropped > 0, "the pass frees log rows");
assert_eq!(store.log_floor().unwrap(), floor);
floor
}
#[tokio::test(flavor = "multi_thread")]
async fn a_puller_below_the_floor_converges_by_merging_a_snapshot_and_deletes_stay_deleted() {
let (daemons, sync_loop) = daemons_with_keys("user@b\n", SHARED_SYNC_KEY, SHARED_SYNC_KEY);
let Daemons {
data_dir,
data_dir_b,
sqlite_a,
sqlite_b,
..
} = &daemons;
let gone = store_note(sqlite_b, "deleted on b before the truncation");
for entry in sqlite_b.sync_log_since(0).unwrap() {
sqlite_a
.apply_record(&second_brain_core::schema::SyncRecord {
local_seq: entry.local_seq as u64,
origin_machine_id: entry.origin_machine_id,
origin_seq: entry.origin_seq as u64,
op: entry.op,
node_type: entry.node_type,
node_id: entry.node_id,
timestamp: entry.timestamp,
data: entry.data.map(|d| serde_json::from_str(&d).unwrap()),
})
.unwrap();
}
for n in 0..4 {
store_note(sqlite_b, &format!("only on b {n}"));
}
sqlite_b.delete_memory(gone.id).unwrap();
store_note(sqlite_b, "the row b keeps in its log");
let own = store_note(sqlite_a, "only on a");
truncate_below(sqlite_b, "machine-c", 1);
assert!(ids_of(sqlite_a).contains(&gone.id));
sync_loop.spawn();
assert!(
wait_until(Duration::from_secs(20), || synced(data_dir.path())).await,
"a never finished a sync"
);
assert_eq!(ids_of(sqlite_a), ids_of(sqlite_b), "the id sets converge");
assert!(!ids_of(sqlite_a).contains(&gone.id), "the delete carried");
assert!(sqlite_a.tombstone_exists(gone.id).unwrap());
assert!(
ids_of(sqlite_b).contains(&own.id),
"a's own write still went out"
);
assert!(
sqlite_a
.get_sync_state("machine-b")
.unwrap()
.is_some_and(|s| s.last_seq >= sqlite_b.log_floor().unwrap()),
"a's cursor for b sits at or above b's floor"
);
assert!(
sqlite_a
.sync_log_since(0)
.unwrap()
.iter()
.any(|e| e.origin_machine_id.starts_with("snap:")),
"the merge logged relays for a's pullers"
);
assert!(
!data_dir.path().join("snapshot.spool").exists(),
"the spool is cleared after the merge"
);
let _ = data_dir_b;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_receiver_below_the_pushers_floor_gets_a_snapshot_and_converges() {
let (daemons, sync_loop) = daemons_with_keys("user@b\n", SHARED_SYNC_KEY, SHARED_SYNC_KEY);
let Daemons {
data_dir_b,
sqlite_a,
sqlite_b,
..
} = &daemons;
for n in 0..5 {
store_note(sqlite_a, &format!("on a before its truncation {n}"));
}
let floor = truncate_below(sqlite_a, "machine-c", 1);
sync_loop.spawn();
assert!(
wait_until(Duration::from_secs(20), || ids_of(sqlite_b)
== ids_of(sqlite_a))
.await,
"b never converged with a"
);
assert!(
wait_until(Duration::from_secs(10), || !data_dir_b
.path()
.join("snapshot.spool")
.exists())
.await,
"b's spool is cleared once merged"
);
assert!(
sqlite_b
.peer_in("machine-a")
.unwrap()
.is_some_and(|seq| seq >= floor),
"b reports peer_in at or past the snapshot's seq"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_record_that_fails_is_kept_and_holds_the_gapless_seq_below_it() {
let dir = tempfile::tempdir().unwrap();
let (app, sqlite) = upgraded("machine-c", dir.path(), SHARED_SYNC_KEY, &[]);
let peer = SqliteStore::in_memory("peer-machine".to_string()).unwrap();
let peer_dir = tempfile::tempdir().unwrap();
upgrade_from_sync_key(&peer, peer_dir.path(), SHARED_SYNC_KEY, &[]);
for n in 0..3 {
store_note(&peer, &format!("from the peer {n}"));
}
let mut exported = Vec::new();
export_changes(&peer, 0, &mut exported).unwrap();
let mut records: Vec<Value> = String::from_utf8(exported)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
records[1]["data"] = serde_json::json!({"not": "a memory"});
let broken_seq = records[1]["local_seq"].as_u64().unwrap();
let last_seq = records[2]["local_seq"].as_u64().unwrap();
let sealer = Keyring::load(&peer, peer_dir.path())
.unwrap()
.sealer_for("machine-c", None)
.unwrap();
let lines: Vec<String> = records
.iter()
.map(|r| seal_line(&sealer, &serde_json::to_vec(r).unwrap()).unwrap())
.collect();
let hello = second_brain_api::sync_hello::hello_for(
&peer,
&ClockGate::open_for_tests(),
peer_dir.path(),
"machine-c",
None,
)
.unwrap();
let import = |lines: Vec<String>| {
authorized(post_json(
"/v1/sync/import",
serde_json::json!({
"header": SyncHeader { hello: Some(hello.clone()), ..SyncHeader::local() },
"lines": lines,
}),
))
};
let body = json_body(app.clone().oneshot(import(lines)).await.unwrap()).await;
assert_eq!(body["reached"], last_seq, "{body}");
assert_eq!(body["max_seq"], broken_seq - 1, "{body}");
assert_eq!(sqlite.failed_record_counts().unwrap().failing, 1);
let again = json_body(app.clone().oneshot(import(Vec::new())).await.unwrap()).await;
assert_eq!(
sqlite.applied_through("peer-machine", last_seq).unwrap(),
broken_seq - 1,
"a retry that fails again keeps holding it: {again}"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_pullers_ack_is_what_it_applied_without_a_gap_not_its_cursor() {
let dir = tempfile::tempdir().unwrap();
let (app, sqlite) = upgraded("machine-c", dir.path(), SHARED_SYNC_KEY, &[]);
for n in 0..3 {
store_note(&sqlite, &format!("from machine-c {n}"));
}
let head = sqlite.log_head().unwrap();
let peer = SqliteStore::in_memory("peer-machine".to_string()).unwrap();
let peer_dir = tempfile::tempdir().unwrap();
upgrade_from_sync_key(&peer, peer_dir.path(), SHARED_SYNC_KEY, &[]);
let hello = Hello {
cursor: Some(head),
applied: Some(1),
..second_brain_api::sync_hello::hello_for(
&peer,
&ClockGate::open_for_tests(),
peer_dir.path(),
"machine-c",
None,
)
.unwrap()
};
let response = app
.oneshot(authorized(post_json(
"/v1/sync/export",
serde_json::json!({
"after_seq": head,
"limit": 10,
"hello": hello,
"header": SyncHeader::local(),
}),
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let ack = sqlite
.peer_acks()
.unwrap()
.into_iter()
.find(|ack| ack.peer_id == "peer-machine")
.and_then(|ack| ack.acked_seq);
assert_eq!(ack, Some(1));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_session_deleted_on_one_daemon_is_gone_from_a_peer_that_still_held_it() {
let (daemons, sync_loop) = daemons_with_keys("user@b\n", SHARED_SYNC_KEY, SHARED_SYNC_KEY);
let Daemons {
app_a,
app_b,
data_dir,
data_dir_b,
sqlite_a,
sqlite_b,
..
} = &daemons;
let session = "5e55a000-0000-4000-8000-0000000000aa";
let path = format!("/home/u/.claude/projects/-srv-garden/{session}.jsonl");
let mut held = Vec::new();
for text in [
"the heron nests behind the boathouse",
"the heron returns each spring",
] {
let mut memory = Memory::new(
text.to_string(),
MemoryType::Episodic,
"claude-code".to_string(),
path.clone(),
)
.with_project_path(Some("-srv-garden".to_string()));
memory.conversation_id = Some(second_brain_core::schema::Conversation::claude_code_id(
session,
));
sqlite_a.store_memory(&memory).unwrap();
sqlite_b.store_memory(&memory).unwrap();
held.push(memory.id);
}
let kept = store_note(sqlite_b, "the shed is painted green");
let response = app_a
.clone()
.oneshot(post_json(
"/v1/delete",
serde_json::json!({"session": session}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(json_body(response).await["removed"]["memories"], 2);
sync_loop.spawn();
assert!(
wait_until(Duration::from_secs(20), || synced(data_dir.path())).await,
"a never finished a sync"
);
assert!(
wait_until(Duration::from_secs(20), || {
let (a, b) = (ids_of(sqlite_a), ids_of(sqlite_b));
held.iter().all(|id| !a.contains(id) && !b.contains(id)) && a.contains(&kept.id)
})
.await,
"the delete reached b and b's copy did not come back to a"
);
for id in &held {
assert!(sqlite_a.tombstone_exists(*id).unwrap());
assert!(sqlite_b.tombstone_exists(*id).unwrap());
}
assert!(sqlite_b.session_deleted(&path).unwrap());
assert!(!recalls(app_a, "heron").await);
assert!(!recalls(app_b, "heron").await);
assert!(
sqlite_b
.sync_log_since(0)
.unwrap()
.iter()
.all(|e| !e.data.as_deref().is_some_and(|d| d.contains("heron"))),
"b's log keeps none of the deleted text"
);
let _ = data_dir_b;
}
async fn delete_status(app: &axum::Router, id: &str) -> Value {
let response = app
.clone()
.oneshot(get(&format!("/v1/delete/{id}")))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
json_body(response).await
}
fn peer_acked(status: &Value, peer: &str) -> Option<bool> {
status["peers"]
.as_array()?
.iter()
.find(|p| p["peer_id"] == peer)
.and_then(|p| p["acked"].as_bool())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_delete_status_shows_the_peer_pending_until_it_acks_past_the_tombstone() {
let (daemons, sync_loop) = daemons_with_keys("user@b\n", SHARED_SYNC_KEY, SHARED_SYNC_KEY);
let Daemons {
app_a,
data_dir: _data_dir,
data_dir_b: _data_dir_b,
sqlite_a,
sqlite_b,
..
} = &daemons;
let gone = store_note(sqlite_a, "the heron nests behind the boathouse");
for peer in ["machine-b", "machine-c"] {
sqlite_a
.record_peer_ack(peer, 0, Utc::now().timestamp_micros())
.unwrap();
}
let response = app_a
.clone()
.oneshot(post_json(
"/v1/delete",
serde_json::json!({"memory": gone.id.to_string()}),
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let id = json_body(response).await["delete_id"]
.as_str()
.unwrap()
.to_string();
let status = delete_status(app_a, &id).await;
assert_eq!(status["scope"], "memory");
assert_eq!(peer_acked(&status, "machine-b"), Some(false), "{status}");
assert_eq!(peer_acked(&status, "machine-c"), Some(false), "{status}");
sync_loop.spawn();
let mut status = Value::Null;
let acked = wait_until(Duration::from_secs(20), || {
status = sqlite_a
.delete_status(&id)
.unwrap()
.map(|s| serde_json::to_value(s).unwrap())
.unwrap_or(Value::Null);
peer_acked(&status, "machine-b") == Some(true)
})
.await;
assert!(acked, "b never acked past the delete: {status}");
let status = delete_status(app_a, &id).await;
assert_eq!(peer_acked(&status, "machine-b"), Some(true), "{status}");
assert_eq!(peer_acked(&status, "machine-c"), Some(false), "{status}");
assert!(sqlite_b.tombstone_exists(gone.id).unwrap());
}