use anyhow::{Context, Result};
use rusqlite::{Connection, ErrorCode, OptionalExtension, Transaction, params};
use serde::{Deserialize, Serialize};
use super::clock::is_new_layout;
use super::floor::DAY_MICROS;
use super::{SqliteStore, sync_dedup};
use crate::schema::SyncRecord;
pub(crate) const SETTLE_AFTER_ATTEMPTS: u32 = 10;
pub(crate) const SETTLE_AFTER_MICROS: i64 = DAY_MICROS;
const FAILED: &str = "sync_failed:";
const REPLAYED_FOR: &str = "sync_settled_replayed_for";
const REPLAY_PAGE: usize = 100;
const DROPPED: &str = "sync_settled_dropped";
pub(crate) const SETTLED_KEEP_MICROS: i64 = 90 * DAY_MICROS;
pub(crate) const SETTLED_CAP: u64 = 10_000;
pub fn replay_build() -> String {
format!(
"{} schema {}",
env!("CARGO_PKG_VERSION"),
super::ddl::SCHEMA_VERSION
)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FailedRecord {
pub source: String,
pub seq: u64,
pub record: SyncRecord,
pub error: String,
pub attempts: u32,
pub first_failed_at: i64,
pub settled: bool,
#[serde(default)]
pub settled_at: Option<i64>,
#[serde(default)]
pub settled_build: Option<String>,
#[serde(default)]
pub replayed_newer: bool,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FailedCounts {
pub failing: u64,
pub settled: u64,
pub dropped: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PruneCounts {
pub aged: u64,
pub capped: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct RetryCounts {
pub applied: u64,
pub settled: u64,
pub failing: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ReplayCounts {
pub recovered: u64,
pub still_failing: u64,
}
pub fn is_transient(error: &anyhow::Error) -> bool {
error.chain().any(|cause| {
if let Some(io) = cause.downcast_ref::<std::io::Error>() {
return !matches!(
io.kind(),
std::io::ErrorKind::Other
| std::io::ErrorKind::InvalidData
| std::io::ErrorKind::InvalidInput
);
}
match cause.downcast_ref::<rusqlite::Error>() {
Some(rusqlite::Error::SqliteFailure(e, _)) => matches!(
e.code,
ErrorCode::DatabaseBusy
| ErrorCode::DatabaseLocked
| ErrorCode::SystemIoFailure
| ErrorCode::DiskFull
| ErrorCode::OutOfMemory
| ErrorCode::CannotOpen
),
_ => false,
}
})
}
pub(super) fn is_outer(error: &anyhow::Error) -> bool {
error.chain().any(|c| c.is::<super::ClockUntrusted>()) || is_transient(error)
}
fn prefix(source: &str) -> String {
format!("{FAILED}{source}:")
}
fn key(source: &str, seq: u64) -> String {
format!("{}{seq:020}", prefix(source))
}
fn prefix_end(source: &str) -> String {
format!("{FAILED}{source};")
}
pub(super) fn put(conn: &Connection, failed: &FailedRecord) -> Result<()> {
conn.execute(
"INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
params![
key(&failed.source, failed.seq),
serde_json::to_string(failed)?
],
)?;
Ok(())
}
fn get(conn: &Connection, source: &str, seq: u64) -> Result<Option<FailedRecord>> {
let value: Option<String> = conn
.query_row(
"SELECT value FROM index_meta WHERE key = ?1",
params![key(source, seq)],
|r| r.get(0),
)
.optional()?;
value
.map(|v| serde_json::from_str(&v).context("reading a failed sync record"))
.transpose()
}
fn remove(conn: &Connection, source: &str, seq: u64) -> Result<()> {
conn.execute(
"DELETE FROM index_meta WHERE key = ?1",
params![key(source, seq)],
)?;
Ok(())
}
fn key_range(source: Option<&str>) -> (String, String) {
match source {
Some(source) => (prefix(source), prefix_end(source)),
None => (
FAILED.to_string(),
format!("{};", FAILED.trim_end_matches(':')),
),
}
}
pub(super) fn drop_failed_of(conn: &Connection, node_id: &str) -> Result<()> {
let (lo, hi) = key_range(None);
conn.execute(
"DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
AND json_extract(value, '$.record.node_id') = ?3",
params![lo, hi, node_id],
)?;
Ok(())
}
fn all_failed(conn: &Connection, source: Option<&str>) -> Result<Vec<FailedRecord>> {
let (lo, hi) = key_range(source);
let mut stmt =
conn.prepare("SELECT value FROM index_meta WHERE key >= ?1 AND key < ?2 ORDER BY key")?;
let rows = stmt.query_map(params![lo, hi], |r| r.get::<_, String>(0))?;
rows.map(|v| serde_json::from_str(&v?).context("reading a failed sync record"))
.collect()
}
fn settled_page(conn: &Connection, after: &str) -> Result<Vec<(String, FailedRecord)>> {
let (_, hi) = key_range(None);
let mut stmt = conn.prepare(
"SELECT key, value FROM index_meta WHERE key > ?1 AND key < ?2 \
AND json_extract(value, '$.settled') IS 1 ORDER BY key LIMIT ?3",
)?;
let rows = stmt.query_map(params![after, hi, REPLAY_PAGE as i64], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
})?;
rows.map(|row| {
let (key, value) = row?;
let failed = serde_json::from_str(&value).context("reading a failed sync record")?;
Ok((key, failed))
})
.collect()
}
pub(super) fn settle_on_tx(
tx: &Transaction<'_>,
source: &str,
seq: u64,
record: &SyncRecord,
error: &anyhow::Error,
now_micros: i64,
) -> Result<()> {
put(
tx,
&FailedRecord {
source: source.to_string(),
seq,
record: record.clone(),
error: format!("{error:#}"),
attempts: 1,
first_failed_at: now_micros,
settled: true,
settled_at: Some(now_micros),
settled_build: Some(replay_build()),
replayed_newer: false,
},
)
}
fn prune_on(tx: &Transaction<'_>, now_micros: i64, cap: u64) -> Result<PruneCounts> {
let (lo, hi) = key_range(None);
let aged = tx.execute(
"DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
AND json_extract(value, '$.settled') IS 1 \
AND json_extract(value, '$.replayed_newer') IS 1 \
AND COALESCE(json_extract(value, '$.settled_at'), \
json_extract(value, '$.first_failed_at')) <= ?3",
params![lo, hi, now_micros - SETTLED_KEEP_MICROS],
)?;
let settled: i64 = tx.query_row(
"SELECT COUNT(*) FROM index_meta WHERE key >= ?1 AND key < ?2 \
AND json_extract(value, '$.settled') IS 1",
params![lo, hi],
|r| r.get(0),
)?;
let over = u64::try_from(settled)?.saturating_sub(cap);
let capped = if over == 0 {
0
} else {
tx.execute(
"DELETE FROM index_meta WHERE key IN (SELECT key FROM index_meta \
WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS 1 \
ORDER BY COALESCE(json_extract(value, '$.settled_at'), \
json_extract(value, '$.first_failed_at')), key \
LIMIT ?3)",
params![lo, hi, i64::try_from(over)?],
)?
};
if capped > 0 {
tx.execute(
"INSERT INTO index_meta (key, value) VALUES (?1, ?2) \
ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + excluded.value",
params![DROPPED, capped as i64],
)?;
}
Ok(PruneCounts {
aged: aged as u64,
capped: capped as u64,
})
}
impl SqliteStore {
pub fn note_failed_record(
&self,
source: &str,
record: &SyncRecord,
error: &anyhow::Error,
now_micros: i64,
) -> Result<()> {
let conn = self.conn();
let tx = conn.unchecked_transaction()?;
let seq = record.local_seq;
let mut failed = get(&tx, source, seq)?.unwrap_or(FailedRecord {
source: source.to_string(),
seq,
record: record.clone(),
error: String::new(),
attempts: 0,
first_failed_at: now_micros,
settled: false,
settled_at: None,
settled_build: None,
replayed_newer: false,
});
failed.attempts = failed.attempts.saturating_add(1);
failed.error = format!("{error:#}");
let was_settled = failed.settled;
failed.settled = !is_transient(error)
&& failed.attempts >= SETTLE_AFTER_ATTEMPTS
&& now_micros - failed.first_failed_at >= SETTLE_AFTER_MICROS;
if failed.settled && !was_settled {
failed.settled_at = Some(now_micros);
failed.settled_build = Some(replay_build());
}
put(&tx, &failed)?;
tx.commit()?;
Ok(())
}
pub fn retry_failed_records(&self, source: &str, now_micros: i64) -> Result<RetryCounts> {
let pending: Vec<FailedRecord> = all_failed(&self.reader(), Some(source))?
.into_iter()
.filter(|f| !f.settled)
.collect();
let mut counts = RetryCounts::default();
for failed in pending {
match crate::store::StoreAdmin::apply_record(self, &failed.record) {
Ok(_) => {
remove(&self.conn(), source, failed.seq)?;
counts.applied += 1;
}
Err(e) if e.chain().any(|c| c.is::<super::ClockUntrusted>()) => return Err(e),
Err(e) => {
self.note_failed_record(source, &failed.record, &e, now_micros)?;
let settled =
get(&self.reader(), source, failed.seq)?.is_some_and(|f| f.settled);
if settled {
counts.settled += 1;
} else {
counts.failing += 1;
}
}
}
}
Ok(counts)
}
pub fn applied_through(&self, source: &str, reached: u64) -> Result<u64> {
let first_failing: Option<i64> = self
.reader()
.query_row(
"SELECT json_extract(value, '$.seq') FROM index_meta \
WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS NOT 1 \
ORDER BY key LIMIT 1",
params![prefix(source), prefix_end(source)],
|r| r.get(0),
)
.optional()?;
Ok(match first_failing {
Some(seq) => reached.min(u64::try_from(seq)?.saturating_sub(1)),
None => reached,
})
}
pub fn failed_records(&self) -> Result<Vec<FailedRecord>> {
all_failed(&self.reader(), None)
}
pub fn failed_record_counts(&self) -> Result<FailedCounts> {
let (lo, hi) = key_range(None);
let (total, settled): (i64, i64) = self.reader().query_row(
"SELECT COUNT(*), COALESCE(SUM(json_extract(value, '$.settled') IS 1), 0) \
FROM index_meta WHERE key >= ?1 AND key < ?2",
params![lo, hi],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
let dropped: Option<i64> = self
.reader()
.query_row(
"SELECT CAST(value AS INTEGER) FROM index_meta WHERE key = ?1",
params![DROPPED],
|r| r.get(0),
)
.optional()?;
Ok(FailedCounts {
failing: u64::try_from(total - settled)?,
settled: u64::try_from(settled)?,
dropped: u64::try_from(dropped.unwrap_or(0))?,
})
}
pub fn replay_settled_records(&self, build: &str) -> Result<Option<ReplayCounts>> {
let done: Option<String> = self
.reader()
.query_row(
"SELECT value FROM index_meta WHERE key = ?1",
params![REPLAYED_FOR],
|r| r.get(0),
)
.optional()?;
if done.as_deref() == Some(build) {
return Ok(None);
}
let replay_id = uuid::Uuid::new_v4().to_string();
let origin = format!("replay:{replay_id}");
let mut counts = ReplayCounts::default();
let mut n = 0u64;
let mut after = FAILED.to_string();
loop {
let page = self.with_txn(|tx| {
let page = settled_page(tx, &after)?;
for (_, failed) in &page {
self.replay_one(tx, failed, build, &origin, &mut n, &mut counts)?;
}
Ok(page)
})?;
match page.last() {
Some((key, _)) if page.len() == REPLAY_PAGE => after.clone_from(key),
_ => break,
}
}
self.conn().execute(
"INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
params![REPLAYED_FOR, build],
)?;
Ok(Some(counts))
}
pub fn prune_settled_records(&self, now_micros: i64) -> Result<PruneCounts> {
self.with_txn(|tx| prune_on(tx, now_micros, SETTLED_CAP))
}
fn replay_one(
&self,
tx: &Transaction<'_>,
failed: &FailedRecord,
build: &str,
origin: &str,
n: &mut u64,
counts: &mut ReplayCounts,
) -> Result<()> {
let relay = relay_record(&failed.record, origin, *n);
match self.apply_logged_on_tx(tx, &relay)? {
Ok(logged) => {
if logged {
*n += 1;
}
let original = &failed.record;
if is_new_layout(&original.origin_machine_id, original.origin_seq as i64) {
sync_dedup::record(
tx,
&original.origin_machine_id,
original.origin_seq as i64,
)?;
}
remove(tx, &failed.source, failed.seq)?;
counts.recovered += 1;
}
Err(_) => {
counts.still_failing += 1;
if !failed.replayed_newer && failed.settled_build.as_deref() != Some(build) {
put(
tx,
&FailedRecord {
replayed_newer: true,
..failed.clone()
},
)?;
}
}
}
Ok(())
}
pub(super) fn apply_logged_on_tx(
&self,
tx: &Transaction<'_>,
record: &SyncRecord,
) -> Result<std::result::Result<bool, anyhow::Error>> {
tx.execute_batch("SAVEPOINT apply_one")?;
match self.apply_on_tx(tx, record) {
Ok(_) => {
let logged = logged(tx, &record.origin_machine_id, record.origin_seq)?;
if logged {
tx.execute_batch("RELEASE apply_one")?;
} else {
tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
}
Ok(Ok(logged))
}
Err(e) => {
tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
if is_outer(&e) {
return Err(e);
}
Ok(Err(e))
}
}
}
}
fn logged(conn: &Connection, origin: &str, origin_seq: u64) -> Result<bool> {
Ok(conn
.query_row(
"SELECT 1 FROM sync_log WHERE id = ?1",
params![format!("{origin}:{origin_seq}")],
|_| Ok(()),
)
.optional()?
.is_some())
}
pub(super) fn relay_record(record: &SyncRecord, origin: &str, n: u64) -> SyncRecord {
let mut relay = record.clone();
if let Some(object) = relay.data.as_mut().and_then(|d| d.as_object_mut()) {
object
.entry(super::authored_order::UPDATED_BY.to_string())
.or_insert_with(|| record.origin_machine_id.clone().into());
}
relay.origin_machine_id = origin.to_string();
relay.origin_seq = n;
relay
}
#[cfg(test)]
mod tests {
use std::io::{Error, ErrorKind};
use super::*;
#[test]
fn an_io_fault_is_transient() {
for kind in [
ErrorKind::StorageFull,
ErrorKind::PermissionDenied,
ErrorKind::TimedOut,
] {
assert!(is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
}
}
#[test]
fn an_io_error_that_describes_bad_content_is_not_transient() {
for kind in [
ErrorKind::Other,
ErrorKind::InvalidData,
ErrorKind::InvalidInput,
] {
assert!(!is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
}
let parse = super::super::parse_relation_type("not_a_relation_type").unwrap_err();
assert!(!is_transient(&anyhow::Error::new(parse)));
}
fn settled_at(source: &str, seq: u64, at: i64) -> FailedRecord {
FailedRecord {
source: source.to_string(),
seq,
record: SyncRecord {
local_seq: seq,
origin_machine_id: source.to_string(),
origin_seq: seq,
op: crate::schema::SyncOp::Create,
node_type: crate::schema::SyncNodeType::Memory,
node_id: format!("node-{seq}"),
timestamp: chrono::Utc::now(),
data: None,
},
error: "bad".to_string(),
attempts: 10,
first_failed_at: at,
settled: true,
settled_at: Some(at),
settled_build: Some(replay_build()),
replayed_newer: false,
}
}
#[test]
fn past_the_cap_the_oldest_settled_records_go_and_are_counted() {
let store = SqliteStore::in_memory("self".to_string()).unwrap();
let conn = store.conn();
for (seq, at) in [(1, 500), (2, 100), (3, 400), (4, 200), (5, 300)] {
put(&conn, &settled_at("peer", seq, at)).unwrap();
}
let mut failing = settled_at("peer", 6, 50);
failing.settled = false;
put(&conn, &failing).unwrap();
drop(conn);
let pruned = store.with_txn(|tx| prune_on(tx, 1_000, 3)).unwrap();
assert_eq!(pruned, PruneCounts { aged: 0, capped: 2 });
let left: Vec<u64> = store
.failed_records()
.unwrap()
.into_iter()
.map(|f| f.seq)
.collect();
assert_eq!(left, vec![1, 3, 5, 6]);
let counts = store.failed_record_counts().unwrap();
assert_eq!((counts.settled, counts.failing, counts.dropped), (3, 1, 2));
}
#[test]
fn counts_read_the_settled_flag_without_decoding_the_record() {
let store =
SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
for (seq, settled) in [(1u64, true), (2, false), (3, true)] {
let value = serde_json::json!({
"settled": settled,
"record": { "op": "written by a later build" },
});
store
.conn()
.execute(
"INSERT INTO index_meta (key, value) VALUES (?1, ?2)",
params![key("peer", seq), value.to_string()],
)
.unwrap();
}
assert_eq!(
store.failed_record_counts().unwrap(),
FailedCounts {
failing: 1,
settled: 2,
dropped: 0,
}
);
}
}