1use anyhow::{Context, Result};
2use rusqlite::{Connection, ErrorCode, OptionalExtension, Transaction, params};
3use serde::{Deserialize, Serialize};
4
5use super::clock::is_new_layout;
6use super::floor::DAY_MICROS;
7use super::{SqliteStore, sync_dedup};
8use crate::schema::SyncRecord;
9
10pub(crate) const SETTLE_AFTER_ATTEMPTS: u32 = 10;
13pub(crate) const SETTLE_AFTER_MICROS: i64 = DAY_MICROS;
14
15const FAILED: &str = "sync_failed:";
16const REPLAYED_FOR: &str = "sync_settled_replayed_for";
17const REPLAY_PAGE: usize = 100;
18const DROPPED: &str = "sync_settled_dropped";
19
20pub(crate) const SETTLED_KEEP_MICROS: i64 = 90 * DAY_MICROS;
23pub(crate) const SETTLED_CAP: u64 = 10_000;
26
27pub fn replay_build() -> String {
30 format!(
31 "{} schema {}",
32 env!("CARGO_PKG_VERSION"),
33 super::ddl::SCHEMA_VERSION
34 )
35}
36
37#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct FailedRecord {
41 pub source: String,
44 pub seq: u64,
46 pub record: SyncRecord,
47 pub error: String,
48 pub attempts: u32,
49 pub first_failed_at: i64,
50 pub settled: bool,
53 #[serde(default)]
54 pub settled_at: Option<i64>,
55 #[serde(default)]
57 pub settled_build: Option<String>,
58 #[serde(default)]
60 pub replayed_newer: bool,
61}
62
63#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
64pub struct FailedCounts {
65 pub failing: u64,
66 pub settled: u64,
67 pub dropped: u64,
69}
70
71#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
72pub struct PruneCounts {
73 pub aged: u64,
74 pub capped: u64,
75}
76
77#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
78pub struct RetryCounts {
79 pub applied: u64,
80 pub settled: u64,
81 pub failing: u64,
82}
83
84#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
85pub struct ReplayCounts {
86 pub recovered: u64,
87 pub still_failing: u64,
88}
89
90pub fn is_transient(error: &anyhow::Error) -> bool {
95 error.chain().any(|cause| {
96 if let Some(io) = cause.downcast_ref::<std::io::Error>() {
97 return !matches!(
98 io.kind(),
99 std::io::ErrorKind::Other
100 | std::io::ErrorKind::InvalidData
101 | std::io::ErrorKind::InvalidInput
102 );
103 }
104 match cause.downcast_ref::<rusqlite::Error>() {
105 Some(rusqlite::Error::SqliteFailure(e, _)) => matches!(
106 e.code,
107 ErrorCode::DatabaseBusy
108 | ErrorCode::DatabaseLocked
109 | ErrorCode::SystemIoFailure
110 | ErrorCode::DiskFull
111 | ErrorCode::OutOfMemory
112 | ErrorCode::CannotOpen
113 ),
114 _ => false,
115 }
116 })
117}
118
119pub(super) fn is_outer(error: &anyhow::Error) -> bool {
122 error.chain().any(|c| c.is::<super::ClockUntrusted>()) || is_transient(error)
123}
124
125fn prefix(source: &str) -> String {
126 format!("{FAILED}{source}:")
127}
128
129fn key(source: &str, seq: u64) -> String {
130 format!("{}{seq:020}", prefix(source))
131}
132
133fn prefix_end(source: &str) -> String {
135 format!("{FAILED}{source};")
136}
137
138pub(super) fn put(conn: &Connection, failed: &FailedRecord) -> Result<()> {
139 conn.execute(
140 "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
141 params![
142 key(&failed.source, failed.seq),
143 serde_json::to_string(failed)?
144 ],
145 )?;
146 Ok(())
147}
148
149fn get(conn: &Connection, source: &str, seq: u64) -> Result<Option<FailedRecord>> {
150 let value: Option<String> = conn
151 .query_row(
152 "SELECT value FROM index_meta WHERE key = ?1",
153 params![key(source, seq)],
154 |r| r.get(0),
155 )
156 .optional()?;
157 value
158 .map(|v| serde_json::from_str(&v).context("reading a failed sync record"))
159 .transpose()
160}
161
162fn remove(conn: &Connection, source: &str, seq: u64) -> Result<()> {
163 conn.execute(
164 "DELETE FROM index_meta WHERE key = ?1",
165 params![key(source, seq)],
166 )?;
167 Ok(())
168}
169
170fn key_range(source: Option<&str>) -> (String, String) {
171 match source {
172 Some(source) => (prefix(source), prefix_end(source)),
173 None => (
174 FAILED.to_string(),
175 format!("{};", FAILED.trim_end_matches(':')),
176 ),
177 }
178}
179
180pub(super) fn drop_failed_of(conn: &Connection, node_id: &str) -> Result<()> {
183 let (lo, hi) = key_range(None);
184 conn.execute(
185 "DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
186 AND json_extract(value, '$.record.node_id') = ?3",
187 params![lo, hi, node_id],
188 )?;
189 Ok(())
190}
191
192fn all_failed(conn: &Connection, source: Option<&str>) -> Result<Vec<FailedRecord>> {
193 let (lo, hi) = key_range(source);
194 let mut stmt =
195 conn.prepare("SELECT value FROM index_meta WHERE key >= ?1 AND key < ?2 ORDER BY key")?;
196 let rows = stmt.query_map(params![lo, hi], |r| r.get::<_, String>(0))?;
197 rows.map(|v| serde_json::from_str(&v?).context("reading a failed sync record"))
198 .collect()
199}
200
201fn settled_page(conn: &Connection, after: &str) -> Result<Vec<(String, FailedRecord)>> {
202 let (_, hi) = key_range(None);
203 let mut stmt = conn.prepare(
204 "SELECT key, value FROM index_meta WHERE key > ?1 AND key < ?2 \
205 AND json_extract(value, '$.settled') IS 1 ORDER BY key LIMIT ?3",
206 )?;
207 let rows = stmt.query_map(params![after, hi, REPLAY_PAGE as i64], |r| {
208 Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
209 })?;
210 rows.map(|row| {
211 let (key, value) = row?;
212 let failed = serde_json::from_str(&value).context("reading a failed sync record")?;
213 Ok((key, failed))
214 })
215 .collect()
216}
217
218pub(super) fn settle_on_tx(
221 tx: &Transaction<'_>,
222 source: &str,
223 seq: u64,
224 record: &SyncRecord,
225 error: &anyhow::Error,
226 now_micros: i64,
227) -> Result<()> {
228 put(
229 tx,
230 &FailedRecord {
231 source: source.to_string(),
232 seq,
233 record: record.clone(),
234 error: format!("{error:#}"),
235 attempts: 1,
236 first_failed_at: now_micros,
237 settled: true,
238 settled_at: Some(now_micros),
239 settled_build: Some(replay_build()),
240 replayed_newer: false,
241 },
242 )
243}
244
245fn prune_on(tx: &Transaction<'_>, now_micros: i64, cap: u64) -> Result<PruneCounts> {
248 let (lo, hi) = key_range(None);
249 let aged = tx.execute(
250 "DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
251 AND json_extract(value, '$.settled') IS 1 \
252 AND json_extract(value, '$.replayed_newer') IS 1 \
253 AND COALESCE(json_extract(value, '$.settled_at'), \
254 json_extract(value, '$.first_failed_at')) <= ?3",
255 params![lo, hi, now_micros - SETTLED_KEEP_MICROS],
256 )?;
257 let settled: i64 = tx.query_row(
258 "SELECT COUNT(*) FROM index_meta WHERE key >= ?1 AND key < ?2 \
259 AND json_extract(value, '$.settled') IS 1",
260 params![lo, hi],
261 |r| r.get(0),
262 )?;
263 let over = u64::try_from(settled)?.saturating_sub(cap);
264 let capped = if over == 0 {
265 0
266 } else {
267 tx.execute(
268 "DELETE FROM index_meta WHERE key IN (SELECT key FROM index_meta \
269 WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS 1 \
270 ORDER BY COALESCE(json_extract(value, '$.settled_at'), \
271 json_extract(value, '$.first_failed_at')), key \
272 LIMIT ?3)",
273 params![lo, hi, i64::try_from(over)?],
274 )?
275 };
276 if capped > 0 {
277 tx.execute(
278 "INSERT INTO index_meta (key, value) VALUES (?1, ?2) \
279 ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + excluded.value",
280 params![DROPPED, capped as i64],
281 )?;
282 }
283 Ok(PruneCounts {
284 aged: aged as u64,
285 capped: capped as u64,
286 })
287}
288
289impl SqliteStore {
290 pub fn note_failed_record(
295 &self,
296 source: &str,
297 record: &SyncRecord,
298 error: &anyhow::Error,
299 now_micros: i64,
300 ) -> Result<()> {
301 let conn = self.conn();
302 let tx = conn.unchecked_transaction()?;
303 let seq = record.local_seq;
304 let mut failed = get(&tx, source, seq)?.unwrap_or(FailedRecord {
305 source: source.to_string(),
306 seq,
307 record: record.clone(),
308 error: String::new(),
309 attempts: 0,
310 first_failed_at: now_micros,
311 settled: false,
312 settled_at: None,
313 settled_build: None,
314 replayed_newer: false,
315 });
316 failed.attempts = failed.attempts.saturating_add(1);
317 failed.error = format!("{error:#}");
318 let was_settled = failed.settled;
319 failed.settled = !is_transient(error)
320 && failed.attempts >= SETTLE_AFTER_ATTEMPTS
321 && now_micros - failed.first_failed_at >= SETTLE_AFTER_MICROS;
322 if failed.settled && !was_settled {
323 failed.settled_at = Some(now_micros);
324 failed.settled_build = Some(replay_build());
325 }
326 put(&tx, &failed)?;
327 tx.commit()?;
328 Ok(())
329 }
330
331 pub fn retry_failed_records(&self, source: &str, now_micros: i64) -> Result<RetryCounts> {
334 let pending: Vec<FailedRecord> = all_failed(&self.reader(), Some(source))?
335 .into_iter()
336 .filter(|f| !f.settled)
337 .collect();
338 let mut counts = RetryCounts::default();
339 for failed in pending {
340 match crate::store::StoreAdmin::apply_record(self, &failed.record) {
341 Ok(_) => {
342 remove(&self.conn(), source, failed.seq)?;
343 counts.applied += 1;
344 }
345 Err(e) if e.chain().any(|c| c.is::<super::ClockUntrusted>()) => return Err(e),
346 Err(e) => {
347 self.note_failed_record(source, &failed.record, &e, now_micros)?;
348 let settled =
349 get(&self.reader(), source, failed.seq)?.is_some_and(|f| f.settled);
350 if settled {
351 counts.settled += 1;
352 } else {
353 counts.failing += 1;
354 }
355 }
356 }
357 }
358 Ok(counts)
359 }
360
361 pub fn applied_through(&self, source: &str, reached: u64) -> Result<u64> {
364 let first_failing: Option<i64> = self
365 .reader()
366 .query_row(
367 "SELECT json_extract(value, '$.seq') FROM index_meta \
368 WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS NOT 1 \
369 ORDER BY key LIMIT 1",
370 params![prefix(source), prefix_end(source)],
371 |r| r.get(0),
372 )
373 .optional()?;
374 Ok(match first_failing {
375 Some(seq) => reached.min(u64::try_from(seq)?.saturating_sub(1)),
376 None => reached,
377 })
378 }
379
380 pub fn failed_records(&self) -> Result<Vec<FailedRecord>> {
381 all_failed(&self.reader(), None)
382 }
383
384 pub fn failed_record_counts(&self) -> Result<FailedCounts> {
385 let (lo, hi) = key_range(None);
386 let (total, settled): (i64, i64) = self.reader().query_row(
387 "SELECT COUNT(*), COALESCE(SUM(json_extract(value, '$.settled') IS 1), 0) \
388 FROM index_meta WHERE key >= ?1 AND key < ?2",
389 params![lo, hi],
390 |r| Ok((r.get(0)?, r.get(1)?)),
391 )?;
392 let dropped: Option<i64> = self
393 .reader()
394 .query_row(
395 "SELECT CAST(value AS INTEGER) FROM index_meta WHERE key = ?1",
396 params![DROPPED],
397 |r| r.get(0),
398 )
399 .optional()?;
400 Ok(FailedCounts {
401 failing: u64::try_from(total - settled)?,
402 settled: u64::try_from(settled)?,
403 dropped: u64::try_from(dropped.unwrap_or(0))?,
404 })
405 }
406
407 pub fn replay_settled_records(&self, build: &str) -> Result<Option<ReplayCounts>> {
415 let done: Option<String> = self
416 .reader()
417 .query_row(
418 "SELECT value FROM index_meta WHERE key = ?1",
419 params![REPLAYED_FOR],
420 |r| r.get(0),
421 )
422 .optional()?;
423 if done.as_deref() == Some(build) {
424 return Ok(None);
425 }
426 let replay_id = uuid::Uuid::new_v4().to_string();
427 let origin = format!("replay:{replay_id}");
428 let mut counts = ReplayCounts::default();
429 let mut n = 0u64;
430 let mut after = FAILED.to_string();
431 loop {
432 let page = self.with_txn(|tx| {
433 let page = settled_page(tx, &after)?;
434 for (_, failed) in &page {
435 self.replay_one(tx, failed, build, &origin, &mut n, &mut counts)?;
436 }
437 Ok(page)
438 })?;
439 match page.last() {
440 Some((key, _)) if page.len() == REPLAY_PAGE => after.clone_from(key),
441 _ => break,
442 }
443 }
444 self.conn().execute(
445 "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
446 params![REPLAYED_FOR, build],
447 )?;
448 Ok(Some(counts))
449 }
450
451 pub fn prune_settled_records(&self, now_micros: i64) -> Result<PruneCounts> {
454 self.with_txn(|tx| prune_on(tx, now_micros, SETTLED_CAP))
455 }
456
457 fn replay_one(
458 &self,
459 tx: &Transaction<'_>,
460 failed: &FailedRecord,
461 build: &str,
462 origin: &str,
463 n: &mut u64,
464 counts: &mut ReplayCounts,
465 ) -> Result<()> {
466 let relay = relay_record(&failed.record, origin, *n);
467 match self.apply_logged_on_tx(tx, &relay)? {
468 Ok(logged) => {
469 if logged {
470 *n += 1;
471 }
472 let original = &failed.record;
473 if is_new_layout(&original.origin_machine_id, original.origin_seq as i64) {
474 sync_dedup::record(
475 tx,
476 &original.origin_machine_id,
477 original.origin_seq as i64,
478 )?;
479 }
480 remove(tx, &failed.source, failed.seq)?;
481 counts.recovered += 1;
482 }
483 Err(_) => {
484 counts.still_failing += 1;
485 if !failed.replayed_newer && failed.settled_build.as_deref() != Some(build) {
486 put(
487 tx,
488 &FailedRecord {
489 replayed_newer: true,
490 ..failed.clone()
491 },
492 )?;
493 }
494 }
495 }
496 Ok(())
497 }
498
499 pub(super) fn apply_logged_on_tx(
504 &self,
505 tx: &Transaction<'_>,
506 record: &SyncRecord,
507 ) -> Result<std::result::Result<bool, anyhow::Error>> {
508 tx.execute_batch("SAVEPOINT apply_one")?;
509 match self.apply_on_tx(tx, record) {
510 Ok(_) => {
511 let logged = logged(tx, &record.origin_machine_id, record.origin_seq)?;
512 if logged {
513 tx.execute_batch("RELEASE apply_one")?;
514 } else {
515 tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
516 }
517 Ok(Ok(logged))
518 }
519 Err(e) => {
520 tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
521 if is_outer(&e) {
522 return Err(e);
523 }
524 Ok(Err(e))
525 }
526 }
527 }
528}
529
530fn logged(conn: &Connection, origin: &str, origin_seq: u64) -> Result<bool> {
531 Ok(conn
532 .query_row(
533 "SELECT 1 FROM sync_log WHERE id = ?1",
534 params![format!("{origin}:{origin_seq}")],
535 |_| Ok(()),
536 )
537 .optional()?
538 .is_some())
539}
540
541pub(super) fn relay_record(record: &SyncRecord, origin: &str, n: u64) -> SyncRecord {
544 let mut relay = record.clone();
545 if let Some(object) = relay.data.as_mut().and_then(|d| d.as_object_mut()) {
546 object
547 .entry(super::authored_order::UPDATED_BY.to_string())
548 .or_insert_with(|| record.origin_machine_id.clone().into());
549 }
550 relay.origin_machine_id = origin.to_string();
551 relay.origin_seq = n;
552 relay
553}
554
555#[cfg(test)]
556mod tests {
557 use std::io::{Error, ErrorKind};
558
559 use super::*;
560
561 #[test]
562 fn an_io_fault_is_transient() {
563 for kind in [
564 ErrorKind::StorageFull,
565 ErrorKind::PermissionDenied,
566 ErrorKind::TimedOut,
567 ] {
568 assert!(is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
569 }
570 }
571
572 #[test]
573 fn an_io_error_that_describes_bad_content_is_not_transient() {
574 for kind in [
575 ErrorKind::Other,
576 ErrorKind::InvalidData,
577 ErrorKind::InvalidInput,
578 ] {
579 assert!(!is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
580 }
581 let parse = super::super::parse_relation_type("not_a_relation_type").unwrap_err();
582 assert!(!is_transient(&anyhow::Error::new(parse)));
583 }
584
585 fn settled_at(source: &str, seq: u64, at: i64) -> FailedRecord {
586 FailedRecord {
587 source: source.to_string(),
588 seq,
589 record: SyncRecord {
590 local_seq: seq,
591 origin_machine_id: source.to_string(),
592 origin_seq: seq,
593 op: crate::schema::SyncOp::Create,
594 node_type: crate::schema::SyncNodeType::Memory,
595 node_id: format!("node-{seq}"),
596 timestamp: chrono::Utc::now(),
597 data: None,
598 },
599 error: "bad".to_string(),
600 attempts: 10,
601 first_failed_at: at,
602 settled: true,
603 settled_at: Some(at),
604 settled_build: Some(replay_build()),
605 replayed_newer: false,
606 }
607 }
608
609 #[test]
610 fn past_the_cap_the_oldest_settled_records_go_and_are_counted() {
611 let store = SqliteStore::in_memory("self".to_string()).unwrap();
612 let conn = store.conn();
613 for (seq, at) in [(1, 500), (2, 100), (3, 400), (4, 200), (5, 300)] {
614 put(&conn, &settled_at("peer", seq, at)).unwrap();
615 }
616 let mut failing = settled_at("peer", 6, 50);
617 failing.settled = false;
618 put(&conn, &failing).unwrap();
619 drop(conn);
620
621 let pruned = store.with_txn(|tx| prune_on(tx, 1_000, 3)).unwrap();
622
623 assert_eq!(pruned, PruneCounts { aged: 0, capped: 2 });
624 let left: Vec<u64> = store
625 .failed_records()
626 .unwrap()
627 .into_iter()
628 .map(|f| f.seq)
629 .collect();
630 assert_eq!(left, vec![1, 3, 5, 6]);
631 let counts = store.failed_record_counts().unwrap();
632 assert_eq!((counts.settled, counts.failing, counts.dropped), (3, 1, 2));
633 }
634
635 #[test]
636 fn counts_read_the_settled_flag_without_decoding_the_record() {
637 let store =
638 SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
639 for (seq, settled) in [(1u64, true), (2, false), (3, true)] {
640 let value = serde_json::json!({
641 "settled": settled,
642 "record": { "op": "written by a later build" },
643 });
644 store
645 .conn()
646 .execute(
647 "INSERT INTO index_meta (key, value) VALUES (?1, ?2)",
648 params![key("peer", seq), value.to_string()],
649 )
650 .unwrap();
651 }
652
653 assert_eq!(
654 store.failed_record_counts().unwrap(),
655 FailedCounts {
656 failing: 1,
657 settled: 2,
658 dropped: 0,
659 }
660 );
661 }
662}