use std::collections::BTreeMap;
use std::fmt;
use anyhow::{Context, Result};
use rusqlite::{Connection, OptionalExtension, Transaction, params};
use super::{SqliteStore, clock, sync_dedup};
use crate::keys::IdentityTag;
use crate::maintenance::is_stale_cursor;
use crate::schema::SyncEntry;
pub(crate) const HOUR_MICROS: i64 = 3_600_000_000;
pub(crate) const DAY_MICROS: i64 = 24 * HOUR_MICROS;
pub(crate) const LIVE_NEIGHBOUR_MICROS: i64 = 30 * DAY_MICROS;
pub(crate) const PIN_LIFETIME_MICROS: i64 = DAY_MICROS;
pub(crate) const DEPARTED_NEIGHBOUR_MICROS: i64 = 90 * DAY_MICROS;
pub(crate) const STUCK_ACK_MICROS: i64 = 7 * DAY_MICROS;
pub const FLOOR_TICK_MICROS: i64 = HOUR_MICROS;
pub(crate) const MAX_UPTIME_STEP_MICROS: i64 = 2 * FLOOR_TICK_MICROS;
pub(crate) const FLOOR_PASS_INTERVAL_MICROS: i64 = DAY_MICROS;
const TRUNCATE_CHUNK: usize = 20_000;
const UPTIME: &str = "floor_uptime";
const TICK_AT: &str = "floor_tick_at";
const PASS_AT: &str = "floor_pass_at";
const LOG_FLOOR: &str = "log_floor";
const ACK_MOVED: &str = "floor_ack_moved:";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Truncation {
Off,
On,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FloorPass {
pub floor: u64,
pub dropped: u64,
pub log_floor: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FloorHealth {
pub floor: u64,
pub log_floor: u64,
pub head: u64,
pub top: u64,
pub silent: Vec<String>,
pub stuck: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PeerAck {
pub peer_id: String,
pub acked_seq: Option<u64>,
pub acked_at: Option<i64>,
pub live: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BelowFloor {
pub after_seq: u64,
pub log_floor: u64,
}
impl fmt::Display for BelowFloor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"below_floor: the sync log holds no rows at or below seq {}, so a read after seq {} \
needs a snapshot",
self.log_floor, self.after_seq
)
}
}
impl std::error::Error for BelowFloor {}
impl SqliteStore {
pub fn advance_floor_uptime(&self, now_micros: i64) -> Result<()> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
if let Some(at) = clock::meta(&tx, TICK_AT)? {
let step = now_micros - at;
if step > 0 && step <= MAX_UPTIME_STEP_MICROS {
clock::set_meta(&tx, UPTIME, uptime(&tx)? + step)?;
}
}
clock::set_meta(&tx, TICK_AT, now_micros)?;
tx.commit()?;
Ok(())
}
pub fn record_peer_ack(&self, peer_id: &str, acked_seq: u64, now_micros: i64) -> Result<()> {
if peer_id == self.machine_id() {
return Ok(());
}
let acked_seq = to_sql(acked_seq)?;
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
let before: Option<Option<i64>> = tx
.query_row(
"SELECT acked_seq FROM peer_acks WHERE peer_id = ?1",
params![peer_id],
|r| r.get(0),
)
.optional()?;
let moved_key = format!("{ACK_MOVED}{peer_id}");
if before.flatten().is_none_or(|before| acked_seq > before)
|| clock::meta(&tx, &moved_key)?.is_none()
{
clock::set_meta(&tx, &moved_key, now_micros)?;
}
tx.execute(
"INSERT INTO peer_acks (peer_id, acked_seq, acked_at, seen_up) VALUES (?1, ?2, ?3, ?4) \
ON CONFLICT(peer_id) DO UPDATE SET \
acked_seq = MAX(COALESCE(peer_acks.acked_seq, 0), excluded.acked_seq), \
acked_at = excluded.acked_at, seen_up = excluded.seen_up",
params![peer_id, acked_seq, now_micros, uptime(&tx)?],
)?;
tx.execute(
"DELETE FROM peer_pins WHERE peer_id = ?1 \
AND seq <= (SELECT acked_seq FROM peer_acks WHERE peer_id = ?1)",
params![peer_id],
)?;
tx.commit()?;
Ok(())
}
pub fn pin_peer(&self, peer_id: &str, seq: u64) -> Result<()> {
if peer_id == self.machine_id() {
return Ok(());
}
let seq = to_sql(seq)?;
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
tx.execute(
"INSERT OR REPLACE INTO peer_pins (peer_id, seq, renewed_up) VALUES (?1, ?2, ?3)",
params![peer_id, seq, uptime(&tx)?],
)?;
tx.commit()?;
Ok(())
}
pub fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
self.conn().execute(
"INSERT INTO peer_in (peer_id, seq) VALUES (?1, ?2) \
ON CONFLICT(peer_id) DO UPDATE SET seq = MAX(peer_in.seq, excluded.seq)",
params![peer_id, to_sql(seq)?],
)?;
Ok(())
}
pub fn peer_in(&self, peer_id: &str) -> Result<Option<u64>> {
let seq: Option<i64> = self
.reader()
.query_row(
"SELECT seq FROM peer_in WHERE peer_id = ?1",
params![peer_id],
|r| r.get(0),
)
.optional()?;
seq.map(from_sql).transpose()
}
pub fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
let reported: Option<i64> = self
.reader()
.query_row(
"SELECT reported FROM push_reports WHERE peer_id = ?1",
params![peer_id],
|r| r.get(0),
)
.optional()?
.flatten();
reported.map(from_sql).transpose()
}
pub fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
self.conn().execute(
"INSERT OR REPLACE INTO push_reports (peer_id, reported) VALUES (?1, ?2)",
params![peer_id, reported.map(to_sql).transpose()?],
)?;
Ok(())
}
pub fn peer_acks(&self) -> Result<Vec<PeerAck>> {
let conn = self.reader();
let up = uptime(&conn)?;
let mut stmt = conn.prepare(
"SELECT peer_id, acked_seq, acked_at, seen_up FROM peer_acks \
WHERE peer_id != ?1 ORDER BY peer_id",
)?;
let rows = stmt.query_map(params![self.machine_id()], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, Option<i64>>(1)?,
r.get::<_, Option<i64>>(2)?,
r.get::<_, i64>(3)?,
))
})?;
rows.map(|row| {
let (peer_id, acked_seq, acked_at, seen_up) = row?;
Ok(PeerAck {
peer_id,
acked_seq: acked_seq.map(from_sql).transpose()?,
acked_at,
live: up - seen_up <= LIVE_NEIGHBOUR_MICROS,
})
})
.collect()
}
pub fn log_floor(&self) -> Result<u64> {
log_floor(&self.reader())
}
pub fn log_head(&self) -> Result<u64> {
head(&self.reader())
}
pub fn floor(&self) -> Result<u64> {
floor(&self.reader(), self.machine_id())
}
pub fn floor_health(&self, now_micros: i64) -> Result<FloorHealth> {
let conn = self.reader();
let tx = conn.unchecked_transaction()?;
let top: i64 = tx.query_row(
"SELECT COALESCE(MAX(local_seq), 0) FROM sync_log",
[],
|r| r.get(0),
)?;
let top = from_sql(top)?;
let up = uptime(&tx)?;
let mut health = FloorHealth {
floor: floor(&tx, self.machine_id())?,
log_floor: log_floor(&tx)?,
head: head(&tx)?,
top,
..FloorHealth::default()
};
let mut stmt = tx.prepare(
"SELECT peer_id, acked_seq, acked_at, seen_up FROM peer_acks \
WHERE peer_id != ?1 ORDER BY peer_id",
)?;
let rows = stmt.query_map(params![self.machine_id()], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, Option<i64>>(1)?,
r.get::<_, Option<i64>>(2)?,
r.get::<_, i64>(3)?,
))
})?;
for row in rows {
let (peer, acked_seq, acked_at, seen_up) = row?;
if up - seen_up > LIVE_NEIGHBOUR_MICROS {
health.silent.push(peer);
continue;
}
let (Some(acked_seq), Some(acked_at)) = (acked_seq, acked_at) else {
continue;
};
let moved_at = clock::meta(&tx, &format!("{ACK_MOVED}{peer}"))?.unwrap_or(now_micros);
if from_sql(acked_seq)? < top
&& now_micros - moved_at >= STUCK_ACK_MICROS
&& now_micros - acked_at < STUCK_ACK_MICROS
{
health.stuck.push(peer);
}
}
drop(stmt);
tx.commit()?;
Ok(health)
}
pub fn floor_pass_due(&self, now_micros: i64) -> Result<bool> {
Ok(match clock::meta(&self.reader(), PASS_AT)? {
Some(at) => now_micros < at || now_micros - at >= FLOOR_PASS_INTERVAL_MICROS,
None => true,
})
}
pub fn run_floor_pass(&self, truncation: Truncation, now_micros: i64) -> Result<FloorPass> {
self.forget_departed_neighbours()?;
let mut dropped = 0u64;
if truncation == Truncation::On {
while let Some(chunk) = self.truncate_chunk()? {
dropped += chunk;
}
}
clock::set_meta(&self.conn(), PASS_AT, now_micros)?;
Ok(FloorPass {
floor: self.floor()?,
dropped,
log_floor: self.log_floor()?,
})
}
fn forget_departed_neighbours(&self) -> Result<()> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
let up = uptime(&tx)?;
tx.execute(
"DELETE FROM peer_acks WHERE peer_id = ?1 \
OR (acked_seq IS NULL AND ?2 - seen_up > ?3) OR ?2 - seen_up > ?4",
params![
self.machine_id(),
up,
LIVE_NEIGHBOUR_MICROS,
DEPARTED_NEIGHBOUR_MICROS
],
)?;
tx.execute(
"DELETE FROM peer_pins WHERE peer_id = ?1 OR ?2 - renewed_up > ?3",
params![self.machine_id(), up, PIN_LIFETIME_MICROS],
)?;
forget_ack_moved_of_dropped(&tx)?;
tx.commit()?;
Ok(())
}
fn truncate_chunk(&self) -> Result<Option<u64>> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
let f = to_sql(floor(&tx, self.machine_id())?)?;
let seqs: Vec<i64> = {
let mut stmt = tx.prepare(
"SELECT local_seq FROM sync_log WHERE local_seq <= ?1 \
ORDER BY local_seq LIMIT ?2",
)?;
let rows = stmt.query_map(params![f, TRUNCATE_CHUNK as i64], |r| r.get(0))?;
rows.collect::<rusqlite::Result<_>>()?
};
let Some(&last) = seqs.last() else {
return Ok(None);
};
sync_dedup::keep_keys_of(&tx, &seqs)?;
let dropped = tx.execute("DELETE FROM sync_log WHERE local_seq <= ?1", params![last])?;
let raised = log_floor(&tx)?.max(from_sql(last)?);
clock::set_meta(&tx, LOG_FLOOR, to_sql(raised)?)?;
tx.commit()?;
Ok(Some(dropped as u64))
}
pub fn seed_floor_neighbours(&self, peers: &[String]) -> Result<usize> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
let up = uptime(&tx)?;
let mut seeded = 0;
{
let mut insert = tx.prepare(
"INSERT OR IGNORE INTO peer_acks (peer_id, acked_seq, acked_at, seen_up) \
VALUES (?1, NULL, NULL, ?2)",
)?;
for peer in peers.iter().filter(|peer| **peer != self.machine_id()) {
seeded += insert.execute(params![peer, up])?;
}
}
tx.commit()?;
Ok(seeded)
}
pub fn reset_floor_for_new_identity(&self, identity_tag: &IdentityTag) -> Result<()> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
for table in ["peer_acks", "peer_pins"] {
tx.execute(
&format!(
"DELETE FROM {table} WHERE peer_id NOT IN \
(SELECT id FROM machines WHERE identity_tag = ?1)"
),
params![identity_tag.as_str()],
)?;
}
forget_ack_moved_of_dropped(&tx)?;
tx.commit()?;
Ok(())
}
pub(super) fn sync_log_page_above_floor(
&self,
after_seq: u64,
limit: Option<usize>,
) -> Result<Vec<SyncEntry>> {
let conn = self.reader();
let tx = conn.unchecked_transaction()?;
let log_floor = log_floor(&tx)?;
if after_seq < log_floor {
return Err(BelowFloor {
after_seq,
log_floor,
}
.into());
}
let page = super::sync_apply::sync_entries_on(&tx, after_seq, limit)?;
tx.commit()?;
Ok(page)
}
}
pub(super) fn seed_neighbours(tx: &Transaction<'_>) -> Result<()> {
let machine_ids = |sql: &str| -> Result<Vec<String>> {
let mut stmt = tx.prepare(sql)?;
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
let mut ids = rows
.collect::<rusqlite::Result<Vec<String>>>()?
.into_iter()
.filter(|key| !is_stale_cursor(key))
.map(|key| key.strip_prefix("push:").unwrap_or(&key).to_string())
.collect::<Vec<_>>();
ids.sort();
ids.dedup();
Ok(ids)
};
let mut neighbours = machine_ids("SELECT peer_id FROM sync_state")?;
if neighbours.is_empty() {
neighbours = machine_ids(DISTINCT_ORIGINS)?;
}
let up = uptime(tx)?;
let mut insert = tx.prepare(
"INSERT OR IGNORE INTO peer_acks (peer_id, acked_seq, acked_at, seen_up) \
VALUES (?1, NULL, NULL, ?2)",
)?;
for peer in neighbours {
insert
.execute(params![peer, up])
.context("seeding a floor neighbour")?;
}
Ok(())
}
const DISTINCT_ORIGINS: &str = "WITH RECURSIVE origins(id) AS ( \
SELECT MIN(origin_machine_id) FROM sync_log \
UNION ALL \
SELECT (SELECT MIN(origin_machine_id) FROM sync_log WHERE origin_machine_id > origins.id) \
FROM origins WHERE origins.id IS NOT NULL \
) SELECT id FROM origins WHERE id IS NOT NULL";
pub(super) fn max_live_ack(conn: &Connection, own_id: &str) -> Result<Option<u64>> {
let up = uptime(conn)?;
let acked: Option<i64> = conn.query_row(
"SELECT MAX(acked_seq) FROM peer_acks WHERE peer_id != ?1 AND ?2 - seen_up <= ?3",
params![own_id, up, LIVE_NEIGHBOUR_MICROS],
|r| r.get(0),
)?;
acked.map(from_sql).transpose()
}
fn forget_ack_moved_of_dropped(tx: &Transaction<'_>) -> Result<()> {
tx.execute(
"DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
AND substr(key, length(?1) + 1) NOT IN (SELECT peer_id FROM peer_acks)",
params![ACK_MOVED, format!("{};", ACK_MOVED.trim_end_matches(':'))],
)?;
Ok(())
}
fn uptime(conn: &Connection) -> Result<i64> {
Ok(clock::meta(conn, UPTIME)?.unwrap_or(0))
}
fn log_floor(conn: &Connection) -> Result<u64> {
clock::meta(conn, LOG_FLOOR)?.map_or(Ok(0), from_sql)
}
fn head(conn: &Connection) -> Result<u64> {
from_sql(clock::state(conn)?.last.max(0))
}
fn floor(conn: &Connection, own_id: &str) -> Result<u64> {
let up = uptime(conn)?;
let mut values: BTreeMap<String, u64> = BTreeMap::new();
{
let mut stmt = conn.prepare(
"SELECT peer_id, COALESCE(acked_seq, 0) FROM peer_acks \
WHERE peer_id != ?1 AND ?2 - seen_up <= ?3",
)?;
let rows = stmt.query_map(params![own_id, up, LIVE_NEIGHBOUR_MICROS], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))
})?;
for row in rows {
let (peer, acked) = row?;
values.insert(peer, from_sql(acked)?);
}
}
let mut stmt = conn.prepare(
"SELECT peer_id, seq FROM peer_pins WHERE peer_id != ?1 AND ?2 - renewed_up <= ?3",
)?;
let rows = stmt.query_map(params![own_id, up, PIN_LIFETIME_MICROS], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))
})?;
for row in rows {
let (peer, seq) = row?;
let seq = from_sql(seq)?;
values
.entry(peer)
.and_modify(|value| *value = (*value).max(seq))
.or_insert(seq);
}
let head = head(conn)?;
Ok(values.into_values().min().unwrap_or(head).min(head))
}
fn to_sql(seq: u64) -> Result<i64> {
i64::try_from(seq).with_context(|| format!("seq {seq} overflows the store's i64"))
}
fn from_sql(seq: i64) -> Result<u64> {
u64::try_from(seq).with_context(|| format!("stored seq {seq} is negative"))
}
#[cfg(test)]
mod tests {
use std::path::Path;
use rusqlite::params;
use super::*;
use crate::schema::{Memory, MemoryType};
use crate::store::{Store, StoreAdmin};
const SELF: &str = "00000000-0000-4000-8000-000000000000";
const A: &str = "00000000-0000-4000-8000-00000000000a";
const B: &str = "00000000-0000-4000-8000-00000000000b";
const C: &str = "00000000-0000-4000-8000-00000000000c";
const T0: i64 = 1_790_000_000_000_000;
fn store() -> SqliteStore {
SqliteStore::in_memory(SELF.to_string()).unwrap()
}
fn write(store: &SqliteStore) -> u64 {
store
.store_memory(&Memory::new(
"floor test".to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
))
.unwrap();
store.sync_log_since(0).unwrap().last().unwrap().local_seq as u64
}
fn log_len(store: &SqliteStore) -> usize {
store.sync_log_since(0).unwrap().len()
}
fn ack_moved(store: &SqliteStore, peer: &str) -> Option<i64> {
clock::meta(&store.reader(), &format!("{ACK_MOVED}{peer}")).unwrap()
}
fn neighbours(store: &SqliteStore) -> Vec<String> {
store
.peer_acks()
.unwrap()
.into_iter()
.map(|ack| ack.peer_id)
.collect()
}
fn run_for(store: &SqliteStore, from: i64, span: i64, each: impl Fn(i64)) -> i64 {
let mut now = from;
store.advance_floor_uptime(now).unwrap();
while now < from + span {
now += FLOOR_TICK_MICROS;
store.advance_floor_uptime(now).unwrap();
each(now);
}
now
}
#[test]
fn f_is_the_head_for_a_single_device_user() {
let s = store();
write(&s);
let head = write(&s);
assert_eq!(s.log_head().unwrap(), head);
assert_eq!(s.floor().unwrap(), head);
}
#[test]
fn the_store_is_never_its_own_neighbour() {
let s = store();
let head = write(&s);
s.record_peer_ack(SELF, 1, T0).unwrap();
s.pin_peer(SELF, 1).unwrap();
assert_eq!(s.floor().unwrap(), head);
assert!(neighbours(&s).is_empty());
}
#[test]
fn f_is_the_lowest_ack_or_pin_over_live_neighbours() {
let s = store();
write(&s);
s.record_peer_ack(A, 5, T0).unwrap();
s.record_peer_ack(B, 9, T0).unwrap();
assert_eq!(s.floor().unwrap(), 5);
s.record_peer_ack(A, 3, T0).unwrap();
assert_eq!(s.floor().unwrap(), 5, "acks never fall");
s.pin_peer(C, 7).unwrap();
s.record_peer_ack(A, 12, T0).unwrap();
assert_eq!(s.floor().unwrap(), 7, "a pin holds F like an ack");
s.record_peer_ack(C, 8, T0).unwrap();
assert_eq!(s.floor().unwrap(), 8, "an ack past the pin replaces it");
s.record_peer_ack(A, 20, T0).unwrap();
s.record_peer_ack(B, 20, T0).unwrap();
assert_eq!(s.floor().unwrap(), 8, "the dropped pin no longer counts");
}
#[test]
fn a_neighbour_silent_for_thirty_days_of_uptime_stops_holding_f() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
s.record_peer_ack(A, 5, T0).unwrap();
s.record_peer_ack(B, 9, T0).unwrap();
run_for(&s, T0, LIVE_NEIGHBOUR_MICROS - FLOOR_TICK_MICROS, |now| {
s.record_peer_ack(B, 9, now).unwrap()
});
assert_eq!(s.floor().unwrap(), 5, "29 days and 23 hours is still live");
let now = T0 + LIVE_NEIGHBOUR_MICROS;
run_for(&s, now, 2 * FLOOR_TICK_MICROS, |now| {
s.record_peer_ack(B, 9, now).unwrap()
});
assert_eq!(s.floor().unwrap(), 9);
let a = s
.peer_acks()
.unwrap()
.into_iter()
.find(|ack| ack.peer_id == A);
assert_eq!(
a.map(|ack| ack.live),
Some(false),
"it stays listed for health"
);
}
#[test]
fn time_the_store_was_off_is_not_silence() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
s.record_peer_ack(A, 5, T0).unwrap();
s.record_peer_ack(B, 9, T0).unwrap();
run_for(&s, T0 + 40 * DAY_MICROS, DAY_MICROS, |_| {});
assert_eq!(s.floor().unwrap(), 5);
}
#[test]
fn a_pin_expires_a_day_of_uptime_after_its_last_renewal() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
s.record_peer_ack(A, 9, T0).unwrap();
s.pin_peer(C, 2).unwrap();
assert_eq!(s.floor().unwrap(), 2);
let now = run_for(&s, T0, 23 * HOUR_MICROS, |_| {});
s.pin_peer(C, 2).unwrap();
run_for(&s, now, 23 * HOUR_MICROS, |_| {});
assert_eq!(
s.floor().unwrap(),
2,
"a renewed pin lives a day from its renewal"
);
let now = run_for(&s, now + 23 * HOUR_MICROS, 2 * HOUR_MICROS, |_| {});
assert_eq!(s.floor().unwrap(), 9);
s.run_floor_pass(Truncation::Off, now).unwrap();
let pins: i64 = s
.conn()
.query_row("SELECT COUNT(*) FROM peer_pins", [], |r| r.get(0))
.unwrap();
assert_eq!(pins, 0, "the pass drops expired pins");
}
#[test]
fn with_truncation_off_the_pass_drops_no_log_row() {
let s = store();
write(&s);
write(&s);
let head = write(&s);
let before = log_len(&s);
let pass = s.run_floor_pass(Truncation::Off, T0).unwrap();
assert_eq!(
pass,
FloorPass {
floor: head,
dropped: 0,
log_floor: 0
}
);
assert_eq!(log_len(&s), before);
assert_eq!(s.log_floor().unwrap(), 0);
}
#[test]
fn with_truncation_on_the_pass_drops_rows_up_to_f_and_keeps_their_keys() {
let s = store();
write(&s);
let second = write(&s);
let third = write(&s);
let dropped_keys: Vec<(String, i64)> = s
.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.local_seq as u64 <= second)
.map(|e| (e.origin_machine_id, e.origin_seq))
.collect();
s.record_peer_ack(A, second, T0).unwrap();
let pass = s.run_floor_pass(Truncation::On, T0).unwrap();
assert_eq!(pass.dropped, 2);
assert_eq!(pass.log_floor, second);
let left: Vec<u64> = s
.sync_log_since(0)
.unwrap()
.iter()
.map(|e| e.local_seq as u64)
.collect();
assert_eq!(left, vec![third]);
let conn = s.conn();
for (origin, origin_seq) in dropped_keys {
assert!(sync_dedup::seen(&conn, &origin, origin_seq).unwrap());
}
}
#[test]
fn a_page_below_the_floor_is_refused_on_any_page() {
let s = store();
let first = write(&s);
let second = write(&s);
let third = write(&s);
s.record_peer_ack(A, second, T0).unwrap();
s.run_floor_pass(Truncation::On, T0).unwrap();
for after_seq in [0, first, second - 1] {
let error = s.sync_log_export_page(after_seq, Some(10)).unwrap_err();
assert_eq!(
error.downcast_ref::<BelowFloor>(),
Some(&BelowFloor {
after_seq,
log_floor: second
})
);
}
let page = s.sync_log_export_page(second, Some(10)).unwrap();
assert_eq!(
page.iter().map(|e| e.local_seq as u64).collect::<Vec<_>>(),
vec![third]
);
}
#[test]
fn the_pass_is_due_once_a_day() {
let s = store();
assert!(s.floor_pass_due(T0).unwrap());
s.run_floor_pass(Truncation::Off, T0).unwrap();
assert!(!s.floor_pass_due(T0 + DAY_MICROS - 1).unwrap());
assert!(s.floor_pass_due(T0 + DAY_MICROS).unwrap());
}
#[test]
fn peer_in_only_rises() {
let s = store();
assert_eq!(s.peer_in(A).unwrap(), None);
s.record_peer_in(A, 7).unwrap();
s.record_peer_in(A, 4).unwrap();
assert_eq!(s.peer_in(A).unwrap(), Some(7));
}
#[test]
fn a_push_report_is_the_last_one_recorded_even_when_lower() {
let s = store();
assert_eq!(s.push_report(A).unwrap(), None);
s.record_push_report(A, Some(5)).unwrap();
s.record_push_report(A, Some(2)).unwrap();
assert_eq!(s.push_report(A).unwrap(), Some(2));
s.record_push_report(A, None).unwrap();
assert_eq!(s.push_report(A).unwrap(), None);
}
fn add_machine(store: &SqliteStore, id: &str, tag: &str) {
store
.conn()
.execute(
"INSERT INTO machines (id, identity_tag) VALUES (?1, ?2)",
params![id, tag],
)
.unwrap();
}
#[test]
fn a_reset_drops_the_old_identitys_neighbours_so_f_is_not_held_at_zero() {
let s = store();
let head = write(&s);
add_machine(&s, A, "m-old");
add_machine(&s, B, "m-new");
s.record_peer_ack(A, 0, T0).unwrap();
s.record_peer_ack(B, 9, T0).unwrap();
s.pin_peer(C, 1).unwrap();
assert_eq!(s.floor().unwrap(), 0);
s.reset_floor_for_new_identity(&IdentityTag::from("m-new".to_string()))
.unwrap();
assert_eq!(neighbours(&s), vec![B.to_string()]);
assert_eq!(s.floor().unwrap(), 9);
assert!(ack_moved(&s, A).is_none(), "the dropped peer's mark goes");
assert!(ack_moved(&s, B).is_some());
s.reset_floor_for_new_identity(&IdentityTag::from("m-newer".to_string()))
.unwrap();
assert_eq!(s.floor().unwrap(), head, "no neighbour left: F is the head");
}
fn store_at_version_11(path: &Path, seed: &str) -> SqliteStore {
{
let conn =
crate::sqlite_store::test_support::create_store_at_version(path, 11).unwrap();
conn.execute_batch(seed).unwrap();
}
SqliteStore::open(path, SELF.to_string()).unwrap()
}
fn log_row(local_seq: i64, origin: &str) -> String {
format!(
"INSERT INTO sync_log (id, local_seq, origin_machine_id, origin_seq, op, node_type, \
node_id, timestamp) VALUES ('{origin}:{local_seq}', {local_seq}, '{origin}', \
{local_seq}, 'create', 'memory', 'n{local_seq}', '2026-01-02T03:04:05Z');"
)
}
#[test]
fn an_initiator_seeds_its_neighbours_from_its_cursors() {
let dir = tempfile::tempdir().unwrap();
let s = store_at_version_11(
&dir.path().join("brain.db"),
&format!(
"INSERT INTO sync_state VALUES ('{A}', 4, '2026-01-02T03:04:05Z');
INSERT INTO sync_state VALUES ('push:{B}', 4, '2026-01-02T03:04:05Z');
INSERT INTO sync_state VALUES ('push:{A}', 4, '2026-01-02T03:04:05Z');
INSERT INTO sync_state VALUES ('user@pi', 4, '2026-01-02T03:04:05Z');
{}",
log_row(1, C)
),
);
assert_eq!(neighbours(&s), vec![A.to_string(), B.to_string()]);
assert_eq!(
s.floor().unwrap(),
0,
"a neighbour that has not acked holds F at 0"
);
}
#[test]
fn a_passive_store_seeds_its_neighbours_from_the_origins_in_its_log() {
let dir = tempfile::tempdir().unwrap();
let s = store_at_version_11(
&dir.path().join("brain.db"),
&[
log_row(1, C),
log_row(2, SELF),
log_row(3, "snap:s1"),
log_row(4, "replay:r1"),
log_row(5, C),
]
.concat(),
);
assert_eq!(neighbours(&s), vec![C.to_string()]);
assert_eq!(s.floor().unwrap(), 0);
}
#[test]
fn neighbours_seeded_from_a_device_list_hold_the_floor_until_they_ack() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
let own = s.machine_id().to_string();
assert_eq!(
s.seed_floor_neighbours(&[A.to_string(), own, A.to_string()])
.unwrap(),
1
);
assert_eq!(neighbours(&s), vec![A.to_string()]);
assert_eq!(s.floor().unwrap(), 0);
s.record_peer_ack(A, 1, T0).unwrap();
assert_eq!(s.seed_floor_neighbours(&[A.to_string()]).unwrap(), 0);
assert_eq!(s.floor().unwrap(), 1);
}
#[test]
fn a_seeded_neighbour_that_never_acks_leaves_after_thirty_days() {
let dir = tempfile::tempdir().unwrap();
let s = store_at_version_11(
&dir.path().join("brain.db"),
&format!(
"INSERT INTO sync_state VALUES ('{A}', 4, '2026-01-02T03:04:05Z');
INSERT INTO sync_state VALUES ('{B}', 4, '2026-01-02T03:04:05Z');"
),
);
write(&s);
s.record_peer_ack(A, 5, T0).unwrap();
let now = run_for(&s, T0, LIVE_NEIGHBOUR_MICROS + FLOOR_TICK_MICROS, |now| {
s.record_peer_ack(A, 5, now).unwrap()
});
s.run_floor_pass(Truncation::Off, now).unwrap();
assert_eq!(neighbours(&s), vec![A.to_string()]);
assert_eq!(s.floor().unwrap(), 5);
}
#[test]
fn health_names_a_neighbour_silent_for_thirty_days_until_it_leaves_at_ninety() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
s.record_peer_ack(A, 1, T0).unwrap();
s.record_peer_ack(B, 1, T0).unwrap();
let keep_b = |now| s.record_peer_ack(B, 1, now).unwrap();
let now = run_for(&s, T0, LIVE_NEIGHBOUR_MICROS - FLOOR_TICK_MICROS, keep_b);
assert!(s.floor_health(now).unwrap().silent.is_empty());
let now = run_for(&s, now, 2 * FLOOR_TICK_MICROS, keep_b);
assert_eq!(s.floor_health(now).unwrap().silent, vec![A.to_string()]);
let now = run_for(
&s,
now,
DEPARTED_NEIGHBOUR_MICROS - LIVE_NEIGHBOUR_MICROS,
keep_b,
);
s.run_floor_pass(Truncation::Off, now).unwrap();
assert_eq!(neighbours(&s), vec![B.to_string()]);
assert!(s.floor_health(now).unwrap().silent.is_empty());
}
#[test]
fn a_departed_neighbour_leaves_no_ack_moved_mark() {
let s = store();
write(&s);
s.advance_floor_uptime(T0).unwrap();
s.record_peer_ack(A, 1, T0).unwrap();
s.record_peer_ack(B, 1, T0).unwrap();
assert!(ack_moved(&s, A).is_some());
let now = run_for(
&s,
T0,
DEPARTED_NEIGHBOUR_MICROS + FLOOR_TICK_MICROS,
|now| s.record_peer_ack(B, 1, now).unwrap(),
);
s.run_floor_pass(Truncation::Off, now).unwrap();
assert_eq!(neighbours(&s), vec![B.to_string()]);
assert!(ack_moved(&s, A).is_none());
assert!(ack_moved(&s, B).is_some());
}
#[test]
fn an_ack_that_stops_moving_while_its_neighbour_keeps_acking_is_stuck() {
let s = store();
let first = write(&s);
write(&s);
s.record_peer_ack(A, first, T0).unwrap();
let fresh = T0 + STUCK_ACK_MICROS - 1;
s.record_peer_ack(A, first, fresh).unwrap();
assert!(s.floor_health(fresh).unwrap().stuck.is_empty());
let later = T0 + STUCK_ACK_MICROS;
s.record_peer_ack(A, first, later).unwrap();
assert_eq!(s.floor_health(later).unwrap().stuck, vec![A.to_string()]);
let top = s.floor_health(later).unwrap().top;
s.record_peer_ack(A, top, later + 1).unwrap();
assert!(s.floor_health(later + 1).unwrap().stuck.is_empty());
}
#[test]
fn a_head_raised_by_a_peer_neither_marks_a_caught_up_ack_stuck_nor_drops_unacked_rows() {
let s = store();
let first = write(&s);
let second = write(&s);
s.raise_clock_floors(&crate::sqlite_store::ClockFloors {
last_seq: Some(second as i64 + 1_000_000),
..Default::default()
})
.unwrap();
assert!(s.log_head().unwrap() > second);
s.record_peer_ack(A, first, T0).unwrap();
s.run_floor_pass(Truncation::On, T0).unwrap();
assert_eq!(log_len(&s), 1, "the row A has not acked stays");
assert_eq!(s.log_floor().unwrap(), first);
s.record_peer_ack(A, second, T0).unwrap();
let later = T0 + 2 * STUCK_ACK_MICROS;
s.record_peer_ack(A, second, later).unwrap();
assert!(
s.floor_health(later).unwrap().stuck.is_empty(),
"an ack at the log's top is caught up whatever the head"
);
}
}