1use std::collections::{BTreeMap, BTreeSet, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17 pub batch_size: usize,
19 pub tombstone_batch: usize,
21 pub ordered_new_publication: bool,
25 pub infer_local_deletions: bool,
29 pub backoff: BackoffPolicy,
30}
31
32impl Default for SyncConfig {
33 fn default() -> Self {
34 Self {
35 batch_size: 200,
36 tombstone_batch: 50,
37 ordered_new_publication: true,
38 infer_local_deletions: false,
39 backoff: BackoffPolicy::default(),
40 }
41 }
42}
43
44#[derive(Debug, Clone, PartialEq)]
45pub enum PassOutcome {
46 Synced,
48 Pending(String),
50 Deferred {
52 secs: f64,
53 },
54 Unavailable(String),
55 Disabled,
56 Cancelled,
58 Failed(String),
59}
60
61#[derive(Debug, Clone, PartialEq)]
62pub struct PassReport {
63 pub outcome: PassOutcome,
64 pub uploaded: usize,
65 pub downloaded: usize,
66 pub tombstones_applied: usize,
67 pub tombstones_written: usize,
68 pub quarantined: usize,
69 pub blocked_reuploads: usize,
70 pub retry_after_secs: Option<f64>,
72}
73
74impl PassReport {
75 fn new() -> Self {
76 Self {
77 outcome: PassOutcome::Synced,
78 uploaded: 0,
79 downloaded: 0,
80 tombstones_applied: 0,
81 tombstones_written: 0,
82 quarantined: 0,
83 blocked_reuploads: 0,
84 retry_after_secs: None,
85 }
86 }
87}
88
89#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
93pub struct SyncMetrics {
94 pub passes: u64,
96 pub failed_passes: u64,
98 pub last_pass_ms: u64,
100 pub downloaded_total: u64,
101 pub uploaded_total: u64,
102 pub tombstones_applied_total: u64,
103 pub tombstones_written_total: u64,
104 pub remote_versions: u64,
106 pub tombstones: u64,
108 pub pending_uploads: u64,
109 pub pending_tombstones: u64,
110 pub pending_apply: u64,
112}
113
114#[derive(Debug, Clone, Copy, PartialEq)]
116pub struct SyncStatus {
117 pub metrics: SyncMetrics,
119 pub process: Option<rightkit_child_process::ProcessUsage>,
123}
124
125#[derive(Debug, Default)]
126struct Counters {
127 passes: u64,
128 failed_passes: u64,
129 last_pass_ms: u64,
130 downloaded: u64,
131 uploaded: u64,
132 tombstones_applied: u64,
133 tombstones_written: u64,
134}
135
136const MAX_CONFLICT_ROUNDS: usize = 4;
137
138enum Resolved {
139 Saved,
141 RemoteWon,
143 Failed(String),
144}
145
146enum RemoteState<T> {
147 Live(T),
148 Tombstone,
149}
150
151struct RemoteEntry<T> {
152 id: String,
153 record_name: String,
154 version_ms: i64,
156 state: RemoteState<T>,
157 record: RawRecord,
158}
159
160impl<T> RemoteEntry<T> {
161 fn is_tombstone(&self) -> bool {
162 matches!(self.state, RemoteState::Tombstone)
163 }
164 fn rank(&self) -> (bool, i64, &str) {
167 (
168 self.is_tombstone(),
169 self.version_ms,
170 self.record_name.as_str(),
171 )
172 }
173}
174
175pub struct SyncEngine<M: RecordMapper> {
176 mapper: Arc<M>,
177 backend: Arc<dyn Backend>,
178 auth: Arc<dyn AuthProvider>,
179 local: Arc<dyn LocalStore<M::Item>>,
180 policy: Box<dyn ConflictPolicy<M::Item>>,
181 clock: Arc<dyn Clock>,
182 observer: Arc<dyn SyncObserver>,
183 hooks: Arc<dyn PassHooks>,
184 config: SyncConfig,
185 store: LedgerStore,
186 replay_flags: Vec<String>,
187 counters: Counters,
188}
189
190pub struct SyncEngineBuilder<M: RecordMapper> {
191 mapper: Arc<M>,
192 backend: Arc<dyn Backend>,
193 local: Arc<dyn LocalStore<M::Item>>,
194 storage: Box<dyn LedgerStorage>,
195 auth: Arc<dyn AuthProvider>,
196 policy: Box<dyn ConflictPolicy<M::Item>>,
197 clock: Arc<dyn Clock>,
198 observer: Arc<dyn SyncObserver>,
199 hooks: Arc<dyn PassHooks>,
200 config: SyncConfig,
201}
202
203impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
204 pub fn new(
205 mapper: Arc<M>,
206 backend: Arc<dyn Backend>,
207 local: Arc<dyn LocalStore<M::Item>>,
208 storage: Box<dyn LedgerStorage>,
209 ) -> Self
210 where
211 M::Item: 'static,
212 {
213 Self {
214 mapper,
215 backend,
216 local,
217 storage,
218 auth: Arc::new(AlwaysAvailable),
219 policy: Box::new(LastWriterWins),
220 clock: Arc::new(SystemClock),
221 observer: Arc::new(NullObserver),
222 hooks: Arc::new(NoHooks),
223 config: SyncConfig::default(),
224 }
225 }
226 pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
227 self.auth = v;
228 self
229 }
230 pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
231 self.policy = v;
232 self
233 }
234 pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
235 self.clock = v;
236 self
237 }
238 pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
239 self.observer = v;
240 self
241 }
242 pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
243 self.hooks = v;
244 self
245 }
246 pub fn config(mut self, v: SyncConfig) -> Self {
247 self.config = v;
248 self
249 }
250 pub fn build(self) -> SyncResult<SyncEngine<M>> {
251 let store = LedgerStore::open(self.storage)?;
252 if store.recovered_from_corruption {
253 self.observer
254 .event("ledger_recovered", "ledger unreadable; full replay");
255 }
256 Ok(SyncEngine {
257 mapper: self.mapper,
258 backend: self.backend,
259 auth: self.auth,
260 local: self.local,
261 policy: self.policy,
262 clock: self.clock,
263 observer: self.observer,
264 hooks: self.hooks,
265 config: self.config,
266 store,
267 replay_flags: Vec::new(),
268 counters: Counters::default(),
269 })
270 }
271}
272
273struct Fetched {
274 records: Vec<RawRecord>,
275 deleted: Vec<String>,
276 token: Option<Vec<u8>>,
277}
278
279impl<M: RecordMapper + 'static> SyncEngine<M>
280where
281 M::Item: 'static,
282{
283 pub fn ledger(&self) -> &crate::ledger::Ledger {
284 self.store.ledger()
285 }
286
287 pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
289 let id = id.to_string();
290 self.store.mutate(|l| {
291 if !l.tombstones.contains_key(&id) {
292 l.pending_uploads.insert(id);
293 }
294 })
295 }
296
297 pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
299 let now = self.clock.now_ms();
300 let id = id.to_string();
301 self.store.mutate(|l| {
302 l.bury(&id, now);
303 l.pending_tombstones.insert(id);
304 })
305 }
306
307 pub fn request_full_replay_once(&mut self, flag: &str) {
309 if !self.store.ledger().flags.contains(flag) {
310 self.replay_flags.push(flag.to_string());
311 }
312 }
313
314 pub fn run_pass(&mut self) -> PassReport {
316 let started = Instant::now();
317 let rep = self.run_pass_once();
318 self.record_pass(&rep, started.elapsed());
319 rep
320 }
321
322 fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
323 let c = &mut self.counters;
324 c.passes += 1;
325 if matches!(
326 rep.outcome,
327 PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
328 ) {
329 c.failed_passes += 1;
330 }
331 c.last_pass_ms = elapsed.as_millis() as u64;
332 c.downloaded += rep.downloaded as u64;
333 c.uploaded += rep.uploaded as u64;
334 c.tombstones_applied += rep.tombstones_applied as u64;
335 c.tombstones_written += rep.tombstones_written as u64;
336 }
337
338 pub fn status(&self) -> SyncStatus {
341 SyncStatus {
342 metrics: self.metrics(),
343 process: rightkit_child_process::process_usage().ok(),
344 }
345 }
346
347 pub fn metrics(&self) -> SyncMetrics {
349 let l = self.store.ledger();
350 let c = &self.counters;
351 SyncMetrics {
352 passes: c.passes,
353 failed_passes: c.failed_passes,
354 last_pass_ms: c.last_pass_ms,
355 downloaded_total: c.downloaded,
356 uploaded_total: c.uploaded,
357 tombstones_applied_total: c.tombstones_applied,
358 tombstones_written_total: c.tombstones_written,
359 remote_versions: l.remote_versions.len() as u64,
360 tombstones: l.tombstones.len() as u64,
361 pending_uploads: l.pending_uploads.len() as u64,
362 pending_tombstones: l.pending_tombstones.len() as u64,
363 pending_apply: l.pending_apply.len() as u64,
364 }
365 }
366
367 fn run_pass_once(&mut self) -> PassReport {
368 let mut rep = PassReport::new();
369 if !self.auth.enabled() {
370 rep.outcome = PassOutcome::Disabled;
371 return rep;
372 }
373 let now = self.clock.now_ms();
374 if let Some(until) = self.store.ledger().retry_after_ms {
375 if until > now {
376 let secs = (until - now) as f64 / 1000.0;
377 rep.outcome = PassOutcome::Deferred { secs };
378 rep.retry_after_secs = Some(secs);
379 return rep;
380 }
381 }
382 let generation = self.auth.generation();
383 match self.pass_inner(generation, &mut rep) {
384 Ok(pending) => {
385 let result = self.store.mutate(|l| {
386 if pending.is_none() {
387 l.consecutive_failures = 0;
388 l.retry_after_ms = None;
389 }
390 });
391 match (result, pending) {
392 (Err(e), _) => self.fail(e, &mut rep),
393 (Ok(()), None) => rep.outcome = PassOutcome::Synced,
394 (Ok(()), Some(why)) => {
395 let secs = self.config.backoff.pending_floor_secs;
396 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
397 let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
398 rep.retry_after_secs = Some(secs);
399 rep.outcome = PassOutcome::Pending(why);
400 }
401 }
402 }
403 Err(e) => self.fail(e, &mut rep),
404 }
405 rep
406 }
407
408 fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
409 if e == SyncError::Cancelled {
410 rep.outcome = PassOutcome::Cancelled;
411 return;
412 }
413 self.observer.event("pass_failed", &e.to_string());
414 let failures = self.store.ledger().consecutive_failures.saturating_add(1);
415 let secs = self
416 .config
417 .backoff
418 .delay_secs(failures, e.retry_hint_secs());
419 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
420 let _ = self.store.mutate(|l| {
421 l.consecutive_failures = failures;
422 l.retry_after_ms = Some(until);
423 });
424 rep.retry_after_secs = Some(secs);
425 rep.outcome = match e {
426 SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
427 other => PassOutcome::Failed(other.to_string()),
428 };
429 }
430
431 fn resolve_conflict(
436 &mut self,
437 header: &ItemHeader,
438 item: &M::Item,
439 name: &str,
440 mut server: Option<RawRecord>,
441 generation: u64,
442 rep: &mut PassReport,
443 ) -> SyncResult<Resolved> {
444 for _ in 0..MAX_CONFLICT_ROUNDS {
445 self.check_current(generation)?;
446 let server_rec = match server.take() {
447 Some(r) => r,
448 None => match self.backend.fetch_record(name)? {
449 Some(r) => r,
450 None => return Ok(Resolved::Failed("conflicting record vanished".into())),
451 },
452 };
453 self.observer.event("save_conflict", name);
454 if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
455 self.local.remove(&id)?;
456 self.store.mutate(|l| l.bury(&id, deleted_at))?;
457 rep.tombstones_applied += 1;
458 return Ok(Resolved::RemoteWon);
459 }
460 let remote_item = match self.mapper.decode(&server_rec) {
461 Ok(i) => i,
462 Err(e) => {
463 rep.quarantined += 1;
464 return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
465 }
466 };
467 let rh = self.mapper.header(&remote_item);
468 if self
469 .policy
470 .upload_local_over_remote(header, item, &rh, &remote_item)
471 {
472 let mut rec = server_rec;
473 rec.set(field::ID, FieldValue::Text(header.id.clone()));
474 rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
475 rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
476 rec.fields.remove(field::DELETED_AT);
477 self.mapper.encode(item, &mut rec);
478 let outcomes = self
479 .backend
480 .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
481 let Some(o) = outcomes.into_iter().next() else {
482 return Ok(Resolved::Failed("no save outcome".into()));
483 };
484 match o.result {
485 Ok(()) => return Ok(Resolved::Saved),
486 Err(SyncError::Conflict) => server = o.server_record,
487 Err(e) => return Ok(Resolved::Failed(e.to_string())),
488 }
489 continue;
490 }
491 let version = RemoteVersion {
492 record_name: name.to_string(),
493 updated_at_ms: rh.updated_at_ms,
494 };
495 if self.policy.accept_remote_over_local(header, &rh) {
496 self.check_current(generation)?;
497 match self.local.apply_remote(&[remote_item])?.into_iter().next() {
498 Some(ApplyOutcome::Deferred(_)) => {
499 let id = header.id.clone();
500 self.store.mutate(|l| {
501 l.pending_apply.insert(id, server_rec);
502 })?;
503 }
504 _ => rep.downloaded += 1,
505 }
506 }
507 let id = header.id.clone();
508 self.store.mutate(|l| {
509 l.pending_uploads.remove(&id);
510 l.remote_versions.insert(id, version);
511 })?;
512 return Ok(Resolved::RemoteWon);
513 }
514 Ok(Resolved::Failed("conflict retries exhausted".into()))
515 }
516
517 fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
521 self.check_current(generation)?;
522 let Some(scope) = self.auth.account_scope() else {
523 return Ok(());
524 };
525 match self.store.ledger().scope.clone() {
526 Some(bound) if bound == scope => Ok(()),
527 None => {
528 let dropped = {
532 let l = self.store.ledger();
533 l.change_token.is_some()
534 || !l.remote_versions.is_empty()
535 || !l.aliases.is_empty()
536 || !l.pending_tombstones.is_empty()
537 };
538 self.store.mutate(|l| {
539 l.scope = Some(scope);
540 l.change_token = None;
541 l.remote_versions.clear();
542 l.aliases.clear();
543 l.pending_tombstones.clear();
544 l.has_synced = false;
545 })?;
546 if dropped {
547 self.observer
548 .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
549 }
550 Ok(())
551 }
552 Some(bound) => {
553 self.hooks.on_account_switch(&bound, &scope);
554 self.store.mutate(|l| {
555 *l = crate::ledger::Ledger::default();
556 l.scope = Some(scope.clone());
557 })?;
558 self.observer.event("account_switched", "ledger reset");
559 Ok(())
560 }
561 }
562 }
563
564 fn check_current(&self, generation: u64) -> SyncResult<()> {
565 if self.auth.enabled() && self.auth.generation() == generation {
566 Ok(())
567 } else {
568 Err(SyncError::Cancelled)
569 }
570 }
571
572 fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
574 self.auth.availability()?;
575 let generation = self.auth.generation().max(generation);
578 self.bind_account(generation)?;
579 self.backend.prepare()?;
580 self.check_current(generation)?;
581
582 let mut extras = self.store.ledger().extras.clone();
584 let outbound = self.hooks.outbound(&mut extras);
585 if !outbound.is_empty() {
586 self.check_current(generation)?;
587 self.write_required(&outbound)?;
588 }
589 self.check_current(generation)?;
590 self.store.mutate(|l| l.extras = extras)?;
591
592 if !self.replay_flags.is_empty() {
594 self.store.mutate(|l| l.change_token = None)?;
595 }
596
597 let fetched = self.fetch_all(generation)?;
598 let mapper_type = self.mapper.record_type().to_string();
599 let mut entries: Vec<RemoteEntry<M::Item>> = Vec::new();
600 let mut foreign: Vec<RawRecord> = Vec::new();
601 for record in fetched.records {
602 if record.record_type != mapper_type {
603 foreign.push(record);
604 continue;
605 }
606 if let Some((id, deleted_at)) = read_tombstone(&record) {
607 entries.push(RemoteEntry {
608 id,
609 record_name: record.record_name.clone(),
610 version_ms: deleted_at,
611 state: RemoteState::Tombstone,
612 record,
613 });
614 continue;
615 }
616 match self.mapper.decode(&record) {
617 Ok(item) => {
618 let h = self.mapper.header(&item);
619 entries.push(RemoteEntry {
620 id: h.id,
621 record_name: record.record_name.clone(),
622 version_ms: h.updated_at_ms,
623 state: RemoteState::Live(item),
624 record,
625 });
626 }
627 Err(why) => {
628 rep.quarantined += 1;
630 self.observer.event(
631 "record_quarantined",
632 &format!("{}: {why}", record.record_name),
633 );
634 }
635 }
636 }
637 for name in &fetched.deleted {
639 let known = self
640 .store
641 .ledger()
642 .aliases
643 .iter()
644 .find(|(_, names)| names.contains(name))
645 .map(|(id, _)| id.clone())
646 .or_else(|| {
647 self.store
648 .ledger()
649 .remote_versions
650 .contains_key(name)
651 .then(|| name.clone())
652 });
653 if let Some(id) = known {
654 if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
655 let mut record = RawRecord::new(&mapper_type, name);
656 let now = self.clock.now_ms();
657 write_tombstone(&mut record, &id, now, &[]);
658 entries.push(RemoteEntry {
659 id,
660 record_name: name.clone(),
661 version_ms: now,
662 state: RemoteState::Tombstone,
663 record,
664 });
665 }
666 }
667 }
668 if rep.quarantined > 0 {
669 let n = rep.quarantined as u64;
670 self.store.mutate(|l| l.quarantined_total += n)?;
671 }
672
673 {
675 let seen: Vec<(String, String)> = entries
676 .iter()
677 .map(|e| (e.id.clone(), e.record_name.clone()))
678 .collect();
679 self.store.mutate(|l| {
680 for (id, name) in seen {
681 l.aliases.entry(id).or_default().insert(name);
682 }
683 })?;
684 }
685
686 let mut canonical: BTreeMap<String, RemoteEntry<M::Item>> = BTreeMap::new();
688 for entry in entries {
689 match canonical.get(&entry.id) {
690 Some(cur) if cur.rank() >= entry.rank() => {}
691 _ => {
692 canonical.insert(entry.id.clone(), entry);
693 }
694 }
695 }
696
697 if !foreign.is_empty() {
699 let mut extras = self.store.ledger().extras.clone();
700 let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
701 if !responses.is_empty() {
707 self.check_current(generation)?;
708 if let Err(e) = self.write_required(&responses) {
709 self.observer.event("foreign_write_failed", &e.to_string());
710 return Err(e);
711 }
712 }
713 self.check_current(generation)?;
714 self.store.mutate(|l| l.extras = extras)?;
715 }
716
717 let tombs: Vec<(String, i64)> = canonical
719 .values()
720 .filter(|e| e.is_tombstone())
721 .map(|e| (e.id.clone(), e.version_ms))
722 .collect();
723 for (id, at) in &tombs {
724 self.check_current(generation)?;
725 self.local.remove(id)?;
726 let (id, at) = (id.clone(), *at);
727 self.store.mutate(|l| {
728 l.bury(&id, at);
729 l.pending_tombstones.remove(&id);
730 })?;
731 }
732 rep.tombstones_applied = tombs.len();
733 self.check_current(generation)?;
734
735 let mut pending: Vec<String> = Vec::new();
736
737 match self.flush_tombstones(&canonical, generation, rep) {
739 Ok(true) => {}
740 Ok(false) => pending.push("tombstones pending".into()),
741 Err(e) => return Err(e),
742 }
743
744 let local_items = self.local.all()?;
746 let mut local_ids: HashSet<String> = HashSet::new();
747 let mut locals: Vec<(ItemHeader, M::Item)> = Vec::new();
748 for item in local_items {
749 let h = self.mapper.header(&item);
750 local_ids.insert(h.id.clone());
751 locals.push((h, item));
752 }
753 if self.config.infer_local_deletions {
754 let gone: Vec<String> = self
755 .store
756 .ledger()
757 .remote_versions
758 .keys()
759 .filter(|id| !local_ids.contains(*id))
760 .filter(|id| !self.store.ledger().tombstones.contains_key(*id))
761 .filter(|id| !self.store.ledger().pending_apply.contains_key(*id))
762 .cloned()
763 .collect();
764 if !gone.is_empty() {
765 let now = self.clock.now_ms();
766 self.store.mutate(|l| {
767 for id in gone {
768 l.bury(&id, now);
769 l.pending_tombstones.insert(id);
770 }
771 })?;
772 if !self.flush_tombstones(&canonical, generation, rep)? {
773 pending.push("tombstones pending".into());
774 }
775 }
776 }
777
778 let blocked: BTreeSet<String> = self.store.ledger().tombstones.keys().cloned().collect();
780 let stale: Vec<String> = self
782 .store
783 .ledger()
784 .pending_uploads
785 .iter()
786 .filter(|id| !local_ids.contains(*id))
787 .cloned()
788 .collect();
789 if !stale.is_empty() {
790 self.store.mutate(|l| {
791 for id in stale {
792 l.pending_uploads.remove(&id);
793 }
794 })?;
795 }
796
797 struct Upload<I> {
798 header: ItemHeader,
799 item: I,
800 base: Option<RawRecord>,
801 is_new: bool,
802 }
803 let mut uploads: Vec<Upload<M::Item>> = Vec::new();
804 let mut failed_uploads = 0usize;
805 for (header, item) in &locals {
806 if blocked.contains(&header.id) {
807 rep.blocked_reuploads += 1;
808 continue;
809 }
810 let queued = self.store.ledger().pending_uploads.contains(&header.id);
811 match canonical.get(&header.id) {
812 Some(RemoteEntry {
813 state: RemoteState::Live(remote_item),
814 record,
815 ..
816 }) => {
817 let rh = self.mapper.header(remote_item);
818 if self
819 .policy
820 .upload_local_over_remote(header, item, &rh, remote_item)
821 {
822 uploads.push(Upload {
823 header: header.clone(),
824 item: item.clone(),
825 base: Some(record.clone()),
826 is_new: false,
827 });
828 }
829 }
830 Some(_) => {}
831 None => {
832 if let Some(known) =
833 self.store.ledger().remote_versions.get(&header.id).cloned()
834 {
835 if known.updated_at_ms < header.updated_at_ms || queued {
837 match self.backend.fetch_record(&known.record_name) {
838 Ok(base) => uploads.push(Upload {
839 header: header.clone(),
840 item: item.clone(),
841 base,
842 is_new: false,
843 }),
844 Err(e) => {
845 failed_uploads += 1;
846 self.observer.event("record_fetch_failed", &e.to_string());
847 }
848 }
849 }
850 } else {
851 uploads.push(Upload {
852 header: header.clone(),
853 item: item.clone(),
854 base: None,
855 is_new: true,
856 });
857 }
858 }
859 }
860 }
861 uploads.sort_by(|a, b| {
863 (
864 a.header.created_at_ms,
865 self.mapper.record_name(&a.header.id),
866 )
867 .cmp(&(
868 b.header.created_at_ms,
869 self.mapper.record_name(&b.header.id),
870 ))
871 });
872 let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
873 let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
874 let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
875 if self.config.ordered_new_publication {
876 chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
877 let mut it = existing.into_iter().peekable();
878 while it.peek().is_some() {
879 chunks.push((false, it.by_ref().take(size).collect()));
880 }
881 } else {
882 let mut it = new_ones.into_iter().chain(existing).peekable();
883 while it.peek().is_some() {
884 chunks.push((false, it.by_ref().take(size).collect()));
885 }
886 }
887
888 let mut blocked_new = false;
889 for (ordered_new, chunk) in chunks {
890 if ordered_new && blocked_new {
891 continue;
892 }
893 self.check_current(generation)?;
894 let mut records = Vec::with_capacity(chunk.len());
895 let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
896 for u in &chunk {
897 let name = self.mapper.record_name(&u.header.id);
898 let mut rec = u
899 .base
900 .clone()
901 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
902 rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
903 rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
904 rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
905 rec.fields.remove(field::DELETED_AT);
906 self.mapper.encode(&u.item, &mut rec);
907 by_name.insert(
908 rec.record_name.clone(),
909 (u.header.id.clone(), u.header.updated_at_ms),
910 );
911 records.push(rec);
912 }
913 let outcomes = self
916 .backend
917 .save_records(&records, SavePolicy::IfServerUnchanged)?;
918 let mut saved: Vec<(String, String, i64)> = Vec::new();
919 let mut chunk_failed = false;
920 for o in outcomes {
921 let Some((id, version)) = by_name.get(&o.record_name) else {
922 continue;
923 };
924 match o.result {
925 Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
926 Err(SyncError::Conflict) => {
927 let upload = chunk
928 .iter()
929 .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
930 .expect("outcome names an upload of this chunk");
931 let (header, item) = (upload.header.clone(), upload.item.clone());
932 match self.resolve_conflict(
933 &header,
934 &item,
935 &o.record_name,
936 o.server_record,
937 generation,
938 rep,
939 )? {
940 Resolved::Saved => {
941 saved.push((id.clone(), o.record_name.clone(), *version))
942 }
943 Resolved::RemoteWon => {}
944 Resolved::Failed(why) => {
945 chunk_failed = true;
946 failed_uploads += 1;
947 self.observer.event(
948 "record_save_failed",
949 &format!("{}: {why}", o.record_name),
950 );
951 }
952 }
953 }
954 Err(e) => {
955 chunk_failed = true;
956 failed_uploads += 1;
957 self.observer
958 .event("record_save_failed", &format!("{}: {e}", o.record_name));
959 }
960 }
961 }
962 if ordered_new && chunk_failed {
963 blocked_new = true;
964 }
965 rep.uploaded += saved.len();
967 self.store.mutate(|l| {
968 for (id, name, version) in saved {
969 l.pending_uploads.remove(&id);
970 l.aliases
971 .entry(id.clone())
972 .or_default()
973 .insert(name.clone());
974 l.remote_versions.insert(
975 id,
976 RemoteVersion {
977 record_name: name,
978 updated_at_ms: version,
979 },
980 );
981 }
982 })?;
983 }
984 if failed_uploads > 0 {
985 pending.push(format!("{failed_uploads} upload(s) pending"));
986 }
987
988 let mut downloads: Vec<(ItemHeader, M::Item, RawRecord)> = Vec::new();
990 let mut seen_ids: HashSet<String> = HashSet::new();
991 for entry in canonical.values() {
992 let RemoteState::Live(item) = &entry.state else {
993 continue;
994 };
995 if self.store.ledger().tombstones.contains_key(&entry.id) {
996 continue;
997 }
998 let rh = self.mapper.header(item);
999 let take = match locals.iter().find(|(h, _)| h.id == entry.id) {
1000 None => true,
1001 Some((lh, _)) => self.policy.accept_remote_over_local(lh, &rh),
1002 };
1003 if take {
1004 seen_ids.insert(entry.id.clone());
1005 downloads.push((rh, item.clone(), entry.record.clone()));
1006 }
1007 }
1008 let stashed: Vec<(String, RawRecord)> = self
1010 .store
1011 .ledger()
1012 .pending_apply
1013 .iter()
1014 .map(|(k, v)| (k.clone(), v.clone()))
1015 .collect();
1016 for (id, record) in stashed {
1017 if seen_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1018 continue;
1019 }
1020 if let Ok(item) = self.mapper.decode(&record) {
1021 downloads.push((self.mapper.header(&item), item, record));
1022 }
1023 }
1024 downloads.sort_by(|a, b| {
1025 (a.0.created_at_ms, a.0.id.as_str()).cmp(&(b.0.created_at_ms, b.0.id.as_str()))
1026 });
1027 if !downloads.is_empty() {
1028 self.check_current(generation)?;
1029 let items: Vec<M::Item> = downloads.iter().map(|d| d.1.clone()).collect();
1030 let outcomes = self.local.apply_remote(&items)?;
1031 if outcomes.len() != downloads.len() {
1032 return Err(SyncError::Persistence(
1033 "apply_remote returned wrong outcome count".into(),
1034 ));
1035 }
1036 let mut deferred = 0usize;
1037 let mut applied_ids: Vec<String> = Vec::new();
1038 let mut stash: Vec<(String, RawRecord)> = Vec::new();
1039 for ((h, _, record), out) in downloads.into_iter().zip(outcomes) {
1040 match out {
1041 ApplyOutcome::Applied => {
1042 rep.downloaded += 1;
1043 applied_ids.push(h.id);
1044 }
1045 ApplyOutcome::Deferred(why) => {
1046 deferred += 1;
1047 self.observer
1048 .event("apply_deferred", &format!("{}: {why}", h.id));
1049 stash.push((h.id, record));
1050 }
1051 }
1052 }
1053 self.store.mutate(|l| {
1054 for id in applied_ids {
1055 l.pending_apply.remove(&id);
1056 }
1057 for (id, rec) in stash {
1058 l.pending_apply.insert(id, rec);
1059 }
1060 })?;
1061 if deferred > 0 {
1062 pending.push(format!("{deferred} download(s) deferred"));
1063 }
1064 }
1065
1066 let versions: Vec<(String, RemoteVersion)> = canonical
1069 .values()
1070 .map(|e| {
1071 (
1072 e.id.clone(),
1073 RemoteVersion {
1074 record_name: e.record_name.clone(),
1075 updated_at_ms: e.version_ms,
1076 },
1077 )
1078 })
1079 .collect();
1080 self.check_current(generation)?;
1083 let flags = std::mem::take(&mut self.replay_flags);
1084 let token = fetched.token;
1085 self.store.mutate(|l| {
1086 for (id, v) in versions {
1087 let keep = l
1089 .remote_versions
1090 .get(&id)
1091 .is_some_and(|k| k.updated_at_ms > v.updated_at_ms);
1092 if !keep {
1093 l.remote_versions.insert(id, v);
1094 }
1095 }
1096 if token.is_some() {
1097 l.change_token = token;
1098 }
1099 l.has_synced = true;
1100 for f in flags {
1101 l.flags.insert(f);
1102 }
1103 })?;
1104 if rep.blocked_reuploads > 0 {
1105 self.observer
1106 .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1107 }
1108 Ok(if pending.is_empty() {
1109 None
1110 } else {
1111 Some(pending.join("; "))
1112 })
1113 }
1114
1115 fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1118 let mut reset_token = false;
1119 let mut recreated = false;
1120 'restart: loop {
1121 let mut token = self.store.ledger().change_token.clone();
1122 let mut out = Fetched {
1123 records: Vec::new(),
1124 deleted: Vec::new(),
1125 token: None,
1126 };
1127 loop {
1128 self.check_current(generation)?;
1129 match self.backend.fetch_changes(token.as_deref()) {
1130 Ok(page) => {
1131 out.records.extend(page.records);
1132 out.deleted.extend(page.deleted_record_names);
1133 if page.new_token.is_some() {
1134 token = page.new_token.clone();
1135 out.token = page.new_token;
1136 }
1137 if !page.more_coming {
1138 return Ok(out);
1139 }
1140 }
1141 Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1142 reset_token = true;
1143 self.observer
1144 .event("token_reset", "change token expired; full replay");
1145 self.store.mutate(|l| l.change_token = None)?;
1146 continue 'restart;
1147 }
1148 Err(SyncError::ZoneMissing) if !recreated => {
1149 recreated = true;
1150 self.observer
1151 .event("zone_recreated", "zone missing; re-preparing");
1152 self.store.mutate(|l| {
1156 l.change_token = None;
1157 l.remote_versions.clear();
1158 l.aliases.clear();
1159 })?;
1160 self.backend.prepare()?;
1161 continue 'restart;
1162 }
1163 Err(e) => return Err(e),
1164 }
1165 }
1166 }
1167 }
1168
1169 fn flush_tombstones(
1172 &mut self,
1173 canonical: &BTreeMap<String, RemoteEntry<M::Item>>,
1174 generation: u64,
1175 rep: &mut PassReport,
1176 ) -> SyncResult<bool> {
1177 let ids: Vec<String> = self
1178 .store
1179 .ledger()
1180 .pending_tombstones
1181 .iter()
1182 .cloned()
1183 .collect();
1184 if ids.is_empty() {
1185 return Ok(true);
1186 }
1187 struct Write {
1188 id: String,
1189 record: RawRecord,
1190 }
1191 let mut writes: Vec<Write> = Vec::new();
1192 let mut never_remote: Vec<String> = Vec::new();
1193 for id in &ids {
1194 let mut names: BTreeSet<String> = self
1195 .store
1196 .ledger()
1197 .aliases
1198 .get(id)
1199 .cloned()
1200 .unwrap_or_default();
1201 let known =
1202 self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1203 if names.is_empty() && !known {
1204 never_remote.push(id.clone());
1205 continue;
1206 }
1207 names.insert(self.mapper.record_name(id));
1208 let at = self
1209 .store
1210 .ledger()
1211 .tombstones
1212 .get(id)
1213 .copied()
1214 .unwrap_or_else(|| self.clock.now_ms());
1215 for name in names {
1216 let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1217 Some(e) => e.record.clone(),
1218 None => self
1219 .backend
1220 .fetch_record(&name)
1221 .ok()
1222 .flatten()
1223 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1224 };
1225 write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1226 writes.push(Write {
1227 id: id.clone(),
1228 record,
1229 });
1230 }
1231 }
1232 if !never_remote.is_empty() {
1233 self.store.mutate(|l| {
1235 for id in never_remote {
1236 l.pending_tombstones.remove(&id);
1237 }
1238 })?;
1239 }
1240 let mut failed: BTreeSet<String> = BTreeSet::new();
1241 let size = self
1242 .config
1243 .tombstone_batch
1244 .min(self.backend.max_batch())
1245 .max(1);
1246 for chunk in writes.chunks(size) {
1247 self.check_current(generation)?;
1248 let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1249 let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1250 for w in chunk {
1251 let ok = outcomes
1252 .iter()
1253 .find(|o| o.record_name == w.record.record_name)
1254 .is_some_and(|o| o.result.is_ok());
1255 if !ok {
1256 failed.insert(w.id.clone());
1257 self.observer
1258 .event("tombstone_save_failed", &w.record.record_name);
1259 }
1260 }
1261 }
1262 let acked: Vec<String> = writes
1263 .iter()
1264 .map(|w| w.id.clone())
1265 .collect::<BTreeSet<_>>()
1266 .into_iter()
1267 .filter(|id| !failed.contains(id))
1268 .collect();
1269 rep.tombstones_written += acked.len();
1270 self.store.mutate(|l| {
1271 for id in &acked {
1272 l.pending_tombstones.remove(id);
1273 }
1274 })?;
1275 Ok(failed.is_empty())
1276 }
1277
1278 fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1280 for chunk in records.chunks(self.backend.max_batch().max(1)) {
1281 for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1282 o.result?;
1283 }
1284 }
1285 Ok(())
1286 }
1287}