1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{ConsentRequest, LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17 pub batch_size: usize,
19 pub tombstone_batch: usize,
21 pub ordered_new_publication: bool,
25 pub infer_local_deletions: bool,
29 pub require_consent_on_scope_change: bool,
35 pub backoff: BackoffPolicy,
36}
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub enum ConsentChoice {
41 Merge,
43 StartFresh,
47}
48
49impl Default for SyncConfig {
50 fn default() -> Self {
51 Self {
52 batch_size: 200,
53 tombstone_batch: 50,
54 ordered_new_publication: true,
55 infer_local_deletions: false,
56 require_consent_on_scope_change: false,
57 backoff: BackoffPolicy::default(),
58 }
59 }
60}
61
62#[derive(Debug, Clone, PartialEq)]
63pub enum PassOutcome {
64 Synced,
66 Pending(String),
68 Deferred {
70 secs: f64,
71 },
72 Unavailable(String),
73 Disabled,
74 ConsentRequired(ConsentRequest),
77 Cancelled,
79 Failed(String),
80}
81
82#[derive(Debug, Clone, PartialEq)]
83pub struct PassReport {
84 pub outcome: PassOutcome,
85 pub uploaded: usize,
86 pub downloaded: usize,
87 pub tombstones_applied: usize,
88 pub tombstones_written: usize,
89 pub quarantined: usize,
90 pub blocked_reuploads: usize,
91 pub retry_after_secs: Option<f64>,
93}
94
95impl PassReport {
96 fn new() -> Self {
97 Self {
98 outcome: PassOutcome::Synced,
99 uploaded: 0,
100 downloaded: 0,
101 tombstones_applied: 0,
102 tombstones_written: 0,
103 quarantined: 0,
104 blocked_reuploads: 0,
105 retry_after_secs: None,
106 }
107 }
108}
109
110#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
114pub struct SyncMetrics {
115 pub passes: u64,
117 pub failed_passes: u64,
119 pub last_pass_ms: u64,
121 pub downloaded_total: u64,
122 pub uploaded_total: u64,
123 pub tombstones_applied_total: u64,
124 pub tombstones_written_total: u64,
125 pub remote_versions: u64,
127 pub tombstones: u64,
129 pub pending_uploads: u64,
130 pub pending_tombstones: u64,
131 pub pending_apply: u64,
133}
134
135#[derive(Debug, Clone, Copy, PartialEq)]
137pub struct SyncStatus {
138 pub metrics: SyncMetrics,
140 #[cfg(feature = "process-usage")]
144 pub process: Option<rightkit_child_process::ProcessUsage>,
145}
146
147#[derive(Debug, Default)]
148struct Counters {
149 passes: u64,
150 failed_passes: u64,
151 last_pass_ms: u64,
152 downloaded: u64,
153 uploaded: u64,
154 tombstones_applied: u64,
155 tombstones_written: u64,
156}
157
158const MAX_CONFLICT_ROUNDS: usize = 4;
159const LOCAL_PAGE_SIZE: usize = 1000;
161const APPLY_CHUNK: usize = 1000;
163
164enum Resolved {
165 Saved,
167 RemoteWon,
169 Failed(String),
170}
171
172enum RemoteState {
173 Live(ItemHeader),
176 Tombstone,
177}
178
179struct RemoteEntry {
180 id: String,
181 record_name: String,
182 version_ms: i64,
184 state: RemoteState,
185 record: RawRecord,
186}
187
188impl RemoteEntry {
189 fn is_tombstone(&self) -> bool {
190 matches!(self.state, RemoteState::Tombstone)
191 }
192 fn rank(&self) -> (bool, i64, &str) {
195 (
196 self.is_tombstone(),
197 self.version_ms,
198 self.record_name.as_str(),
199 )
200 }
201}
202
203pub struct SyncEngine<M: RecordMapper> {
204 mapper: Arc<M>,
205 backend: Arc<dyn Backend>,
206 auth: Arc<dyn AuthProvider>,
207 local: Arc<dyn LocalStore<M::Item>>,
208 policy: Box<dyn ConflictPolicy<M::Item>>,
209 clock: Arc<dyn Clock>,
210 observer: Arc<dyn SyncObserver>,
211 hooks: Arc<dyn PassHooks>,
212 config: SyncConfig,
213 store: LedgerStore,
214 replay_flags: Vec<String>,
215 counters: Counters,
216 gate: Option<ConsentRequest>,
218 local_page_size: usize,
219}
220
221pub struct SyncEngineBuilder<M: RecordMapper> {
222 mapper: Arc<M>,
223 backend: Arc<dyn Backend>,
224 local: Arc<dyn LocalStore<M::Item>>,
225 storage: Box<dyn LedgerStorage>,
226 auth: Arc<dyn AuthProvider>,
227 policy: Box<dyn ConflictPolicy<M::Item>>,
228 clock: Arc<dyn Clock>,
229 observer: Arc<dyn SyncObserver>,
230 hooks: Arc<dyn PassHooks>,
231 config: SyncConfig,
232 local_page_size: usize,
233}
234
235impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
236 pub fn new(
237 mapper: Arc<M>,
238 backend: Arc<dyn Backend>,
239 local: Arc<dyn LocalStore<M::Item>>,
240 storage: Box<dyn LedgerStorage>,
241 ) -> Self
242 where
243 M::Item: 'static,
244 {
245 Self {
246 mapper,
247 backend,
248 local,
249 storage,
250 auth: Arc::new(AlwaysAvailable),
251 policy: Box::new(LastWriterWins),
252 clock: Arc::new(SystemClock),
253 observer: Arc::new(NullObserver),
254 hooks: Arc::new(NoHooks),
255 config: SyncConfig::default(),
256 local_page_size: LOCAL_PAGE_SIZE,
257 }
258 }
259 pub fn local_page_size(mut self, n: usize) -> Self {
261 self.local_page_size = n.max(1);
262 self
263 }
264 pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
265 self.auth = v;
266 self
267 }
268 pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
269 self.policy = v;
270 self
271 }
272 pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
273 self.clock = v;
274 self
275 }
276 pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
277 self.observer = v;
278 self
279 }
280 pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
281 self.hooks = v;
282 self
283 }
284 pub fn config(mut self, v: SyncConfig) -> Self {
285 self.config = v;
286 self
287 }
288 pub fn build(self) -> SyncResult<SyncEngine<M>> {
289 let store = LedgerStore::open(self.storage)?;
290 if store.recovered_from_corruption {
291 self.observer
292 .event("ledger_recovered", "ledger unreadable; full replay");
293 }
294 Ok(SyncEngine {
295 mapper: self.mapper,
296 backend: self.backend,
297 auth: self.auth,
298 local: self.local,
299 policy: self.policy,
300 clock: self.clock,
301 observer: self.observer,
302 hooks: self.hooks,
303 config: self.config,
304 store,
305 replay_flags: Vec::new(),
306 counters: Counters::default(),
307 gate: None,
308 local_page_size: self.local_page_size,
309 })
310 }
311}
312
313struct Fetched {
314 records: Vec<RawRecord>,
315 deleted: Vec<String>,
316 token: Option<Vec<u8>>,
317}
318
319impl<M: RecordMapper + 'static> SyncEngine<M>
320where
321 M::Item: 'static,
322{
323 pub fn ledger(&self) -> &crate::ledger::Ledger {
324 self.store.ledger()
325 }
326
327 pub fn consent_request(&self) -> Option<ConsentRequest> {
330 self.store.ledger().pending_consent.clone()
331 }
332
333 pub fn grant_consent(&mut self, to: &str, choice: ConsentChoice) -> SyncResult<()> {
339 let Some(req) = self.store.ledger().pending_consent.clone() else {
340 return Err(SyncError::Transient(
341 "no scope change is awaiting consent".into(),
342 ));
343 };
344 if req.to != to {
345 return Err(SyncError::Transient(
346 "consent names a different account than the one pending".into(),
347 ));
348 }
349 if let Some(current) = self.auth.account_scope() {
350 if current != req.to {
351 return Err(SyncError::Transient(
352 "the signed-in account changed again; consent is stale".into(),
353 ));
354 }
355 }
356 let withheld: BTreeSet<String> = match choice {
357 ConsentChoice::Merge => BTreeSet::new(),
358 ConsentChoice::StartFresh => {
359 let mut ids = BTreeSet::new();
360 self.scan_local(|i| {
361 ids.insert(self.mapper.header(&i).id);
362 Ok(())
363 })?;
364 ids
365 }
366 };
367 self.hooks.on_account_switch(&req.from, &req.to);
368 let scope = req.to.clone();
369 self.store.mutate(|l| {
370 *l = crate::ledger::Ledger::default();
371 l.scope = Some(scope);
372 l.withheld = withheld;
373 })?;
374 self.gate = None;
375 self.observer.event(
376 "consent_granted",
377 match choice {
378 ConsentChoice::Merge => "merge",
379 ConsentChoice::StartFresh => "start_fresh",
380 },
381 );
382 Ok(())
383 }
384
385 fn scan_local(&self, mut each: impl FnMut(M::Item) -> SyncResult<()>) -> SyncResult<()> {
389 let size = self.local_page_size.max(1);
390 let mut cursor: Option<String> = None;
391 loop {
392 let (page, last) = match self.local.list_page(cursor.as_deref(), size)? {
393 Some(p) => {
394 let last = p.len() < size;
395 (p, last)
396 }
397 None => (self.local.all()?, true),
398 };
399 let next = if last {
400 None
401 } else {
402 page.last().map(|i| self.mapper.header(i).id)
403 };
404 if let (Some(n), Some(c)) = (next.as_ref(), cursor.as_ref()) {
405 if n <= c {
406 return Err(SyncError::Persistence(
407 "LocalStore::list_page did not advance past its cursor".into(),
408 ));
409 }
410 }
411 for item in page {
412 each(item)?;
413 }
414 match next {
415 Some(n) => cursor = Some(n),
416 None => return Ok(()),
417 }
418 }
419 }
420
421 pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
423 let id = id.to_string();
424 self.store.mutate(|l| {
425 if !l.tombstones.contains_key(&id) {
426 l.pending_uploads.insert(id);
427 }
428 })
429 }
430
431 pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
433 let now = self.clock.now_ms();
434 let id = id.to_string();
435 self.store.mutate(|l| {
436 l.bury(&id, now);
437 l.pending_tombstones.insert(id);
438 })
439 }
440
441 pub fn request_full_replay_once(&mut self, flag: &str) {
443 if !self.store.ledger().flags.contains(flag) {
444 self.replay_flags.push(flag.to_string());
445 }
446 }
447
448 pub fn run_pass(&mut self) -> PassReport {
450 let started = Instant::now();
451 let rep = self.run_pass_once();
452 self.record_pass(&rep, started.elapsed());
453 rep
454 }
455
456 fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
457 let c = &mut self.counters;
458 c.passes += 1;
459 if matches!(
460 rep.outcome,
461 PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
462 ) {
463 c.failed_passes += 1;
464 }
465 c.last_pass_ms = elapsed.as_millis() as u64;
466 c.downloaded += rep.downloaded as u64;
467 c.uploaded += rep.uploaded as u64;
468 c.tombstones_applied += rep.tombstones_applied as u64;
469 c.tombstones_written += rep.tombstones_written as u64;
470 }
471
472 pub fn status(&self) -> SyncStatus {
475 SyncStatus {
476 metrics: self.metrics(),
477 #[cfg(feature = "process-usage")]
478 process: rightkit_child_process::process_usage().ok(),
479 }
480 }
481
482 pub fn metrics(&self) -> SyncMetrics {
484 let l = self.store.ledger();
485 let c = &self.counters;
486 SyncMetrics {
487 passes: c.passes,
488 failed_passes: c.failed_passes,
489 last_pass_ms: c.last_pass_ms,
490 downloaded_total: c.downloaded,
491 uploaded_total: c.uploaded,
492 tombstones_applied_total: c.tombstones_applied,
493 tombstones_written_total: c.tombstones_written,
494 remote_versions: l.remote_versions.len() as u64,
495 tombstones: l.tombstones.len() as u64,
496 pending_uploads: l.pending_uploads.len() as u64,
497 pending_tombstones: l.pending_tombstones.len() as u64,
498 pending_apply: l.pending_apply.len() as u64,
499 }
500 }
501
502 fn run_pass_once(&mut self) -> PassReport {
503 let mut rep = PassReport::new();
504 if !self.auth.enabled() {
505 rep.outcome = PassOutcome::Disabled;
506 return rep;
507 }
508 let now = self.clock.now_ms();
509 if let Some(until) = self.store.ledger().retry_after_ms {
510 if until > now {
511 let secs = (until - now) as f64 / 1000.0;
512 rep.outcome = PassOutcome::Deferred { secs };
513 rep.retry_after_secs = Some(secs);
514 return rep;
515 }
516 }
517 let generation = self.auth.generation();
518 self.gate = None;
519 let result = self.pass_inner(generation, &mut rep);
520 if let Some(req) = self.gate.take() {
521 if result.is_ok() {
522 rep.outcome = PassOutcome::ConsentRequired(req);
523 return rep;
524 }
525 }
526 match result {
527 Ok(pending) => {
528 let result = self.store.mutate(|l| {
529 if pending.is_none() {
530 l.consecutive_failures = 0;
531 l.retry_after_ms = None;
532 }
533 });
534 match (result, pending) {
535 (Err(e), _) => self.fail(e, &mut rep),
536 (Ok(()), None) => rep.outcome = PassOutcome::Synced,
537 (Ok(()), Some(why)) => {
538 let secs = self.config.backoff.pending_floor_secs;
539 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
540 let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
541 rep.retry_after_secs = Some(secs);
542 rep.outcome = PassOutcome::Pending(why);
543 }
544 }
545 }
546 Err(e) => self.fail(e, &mut rep),
547 }
548 rep
549 }
550
551 fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
552 if e == SyncError::Cancelled {
553 rep.outcome = PassOutcome::Cancelled;
554 return;
555 }
556 self.observer.event("pass_failed", &e.to_string());
557 let failures = self.store.ledger().consecutive_failures.saturating_add(1);
558 let secs = self
559 .config
560 .backoff
561 .delay_secs(failures, e.retry_hint_secs());
562 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
563 let _ = self.store.mutate(|l| {
564 l.consecutive_failures = failures;
565 l.retry_after_ms = Some(until);
566 });
567 rep.retry_after_secs = Some(secs);
568 rep.outcome = match e {
569 SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
570 other => PassOutcome::Failed(other.to_string()),
571 };
572 }
573
574 fn resolve_conflict(
579 &mut self,
580 header: &ItemHeader,
581 item: &M::Item,
582 name: &str,
583 mut server: Option<RawRecord>,
584 generation: u64,
585 rep: &mut PassReport,
586 ) -> SyncResult<Resolved> {
587 for _ in 0..MAX_CONFLICT_ROUNDS {
588 self.check_current(generation)?;
589 let server_rec = match server.take() {
590 Some(r) => r,
591 None => match self.backend.fetch_record(name)? {
592 Some(r) => r,
593 None => return Ok(Resolved::Failed("conflicting record vanished".into())),
594 },
595 };
596 self.observer.event("save_conflict", name);
597 if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
598 self.local.remove(&id)?;
599 self.store.mutate(|l| l.bury(&id, deleted_at))?;
600 rep.tombstones_applied += 1;
601 return Ok(Resolved::RemoteWon);
602 }
603 let remote_item = match self.mapper.decode(&server_rec) {
604 Ok(i) => i,
605 Err(e) => {
606 rep.quarantined += 1;
607 return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
608 }
609 };
610 let rh = self.mapper.header(&remote_item);
611 if self
612 .policy
613 .upload_local_over_remote(header, item, &rh, &remote_item)
614 {
615 let mut rec = server_rec;
616 rec.set(field::ID, FieldValue::Text(header.id.clone()));
617 rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
618 rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
619 rec.fields.remove(field::DELETED_AT);
620 self.mapper.encode(item, &mut rec);
621 let outcomes = self
622 .backend
623 .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
624 let Some(o) = outcomes.into_iter().next() else {
625 return Ok(Resolved::Failed("no save outcome".into()));
626 };
627 match o.result {
628 Ok(()) => return Ok(Resolved::Saved),
629 Err(SyncError::Conflict) => server = o.server_record,
630 Err(e) => return Ok(Resolved::Failed(e.to_string())),
631 }
632 continue;
633 }
634 let version = RemoteVersion {
635 record_name: name.to_string(),
636 updated_at_ms: rh.updated_at_ms,
637 };
638 if self.policy.accept_remote_over_local(header, &rh) {
639 self.check_current(generation)?;
640 match self.local.apply_remote(&[remote_item])?.into_iter().next() {
641 Some(ApplyOutcome::Deferred(_)) => {
642 let id = header.id.clone();
643 self.store.mutate(|l| {
644 l.pending_apply.insert(id, server_rec);
645 })?;
646 }
647 _ => rep.downloaded += 1,
648 }
649 }
650 let id = header.id.clone();
651 self.store.mutate(|l| {
652 l.pending_uploads.remove(&id);
653 l.remote_versions.insert(id, version);
654 })?;
655 return Ok(Resolved::RemoteWon);
656 }
657 Ok(Resolved::Failed("conflict retries exhausted".into()))
658 }
659
660 fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
668 self.check_current(generation)?;
669 let Some(scope) = self.auth.account_scope() else {
670 return Ok(());
671 };
672 match self.store.ledger().scope.clone() {
673 Some(bound) if bound == scope => {
674 if self.store.ledger().pending_consent.is_some() {
675 self.store.mutate(|l| l.pending_consent = None)?;
677 }
678 Ok(())
679 }
680 Some(bound) if self.config.require_consent_on_scope_change => {
681 let req = ConsentRequest {
682 from: bound,
683 to: scope,
684 };
685 if self.store.ledger().pending_consent.as_ref() != Some(&req) {
686 let stored = req.clone();
687 self.store.mutate(|l| l.pending_consent = Some(stored))?;
688 self.observer
689 .event("consent_required", &format!("{} -> {}", req.from, req.to));
690 }
691 self.gate = Some(req);
692 Ok(())
693 }
694 None => {
695 let dropped = {
699 let l = self.store.ledger();
700 l.change_token.is_some()
701 || !l.remote_versions.is_empty()
702 || !l.aliases.is_empty()
703 || !l.pending_tombstones.is_empty()
704 };
705 self.store.mutate(|l| {
706 l.scope = Some(scope);
707 l.change_token = None;
708 l.remote_versions.clear();
709 l.aliases.clear();
710 l.pending_tombstones.clear();
711 l.has_synced = false;
712 })?;
713 if dropped {
714 self.observer
715 .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
716 }
717 Ok(())
718 }
719 Some(bound) => {
720 self.hooks.on_account_switch(&bound, &scope);
721 self.store.mutate(|l| {
722 *l = crate::ledger::Ledger::default();
723 l.scope = Some(scope.clone());
724 })?;
725 self.observer.event("account_switched", "ledger reset");
726 Ok(())
727 }
728 }
729 }
730
731 fn check_current(&self, generation: u64) -> SyncResult<()> {
732 if self.auth.enabled() && self.auth.generation() == generation {
733 Ok(())
734 } else {
735 Err(SyncError::Cancelled)
736 }
737 }
738
739 fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
741 self.auth.availability()?;
742 let generation = self.auth.generation().max(generation);
745 self.bind_account(generation)?;
746 if self.gate.is_some() {
747 return Ok(None);
748 }
749 self.backend.prepare()?;
750 self.check_current(generation)?;
751
752 let mut extras = self.store.ledger().extras.clone();
754 let outbound = self.hooks.outbound(&mut extras);
755 if !outbound.is_empty() {
756 self.check_current(generation)?;
757 self.write_required(&outbound)?;
758 }
759 self.check_current(generation)?;
760 self.store.mutate(|l| l.extras = extras)?;
761
762 if !self.replay_flags.is_empty() {
764 self.store.mutate(|l| l.change_token = None)?;
765 }
766
767 let fetched = self.fetch_all(generation)?;
768 let mapper_type = self.mapper.record_type().to_string();
769 let mut entries: Vec<RemoteEntry> = Vec::new();
770 let mut foreign: Vec<RawRecord> = Vec::new();
771 for record in fetched.records {
772 if record.record_type != mapper_type {
773 foreign.push(record);
774 continue;
775 }
776 if let Some((id, deleted_at)) = read_tombstone(&record) {
777 entries.push(RemoteEntry {
778 id,
779 record_name: record.record_name.clone(),
780 version_ms: deleted_at,
781 state: RemoteState::Tombstone,
782 record,
783 });
784 continue;
785 }
786 match self.mapper.decode(&record) {
787 Ok(item) => {
788 let h = self.mapper.header(&item);
789 entries.push(RemoteEntry {
790 id: h.id.clone(),
791 record_name: record.record_name.clone(),
792 version_ms: h.updated_at_ms,
793 state: RemoteState::Live(h),
794 record,
795 });
796 }
797 Err(why) => {
798 rep.quarantined += 1;
800 self.observer.event(
801 "record_quarantined",
802 &format!("{}: {why}", record.record_name),
803 );
804 }
805 }
806 }
807 for name in &fetched.deleted {
809 let known = self
810 .store
811 .ledger()
812 .aliases
813 .iter()
814 .find(|(_, names)| names.contains(name))
815 .map(|(id, _)| id.clone())
816 .or_else(|| {
817 self.store
818 .ledger()
819 .remote_versions
820 .contains_key(name)
821 .then(|| name.clone())
822 });
823 if let Some(id) = known {
824 if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
825 let mut record = RawRecord::new(&mapper_type, name);
826 let now = self.clock.now_ms();
827 write_tombstone(&mut record, &id, now, &[]);
828 entries.push(RemoteEntry {
829 id,
830 record_name: name.clone(),
831 version_ms: now,
832 state: RemoteState::Tombstone,
833 record,
834 });
835 }
836 }
837 }
838 if rep.quarantined > 0 {
839 let n = rep.quarantined as u64;
840 self.store.mutate(|l| l.quarantined_total += n)?;
841 }
842
843 self.store.mutate(|l| {
845 for e in &entries {
846 l.aliases
847 .entry(e.id.clone())
848 .or_default()
849 .insert(e.record_name.clone());
850 }
851 })?;
852
853 let mut canonical: BTreeMap<String, RemoteEntry> = BTreeMap::new();
855 for entry in entries {
856 match canonical.get(&entry.id) {
857 Some(cur) if cur.rank() >= entry.rank() => {}
858 _ => {
859 canonical.insert(entry.id.clone(), entry);
860 }
861 }
862 }
863
864 if !foreign.is_empty() {
866 let mut extras = self.store.ledger().extras.clone();
867 let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
868 if !responses.is_empty() {
874 self.check_current(generation)?;
875 if let Err(e) = self.write_required(&responses) {
876 self.observer.event("foreign_write_failed", &e.to_string());
877 return Err(e);
878 }
879 }
880 self.check_current(generation)?;
881 self.store.mutate(|l| l.extras = extras)?;
882 }
883
884 let tombs: Vec<(String, i64)> = canonical
886 .values()
887 .filter(|e| e.is_tombstone())
888 .map(|e| (e.id.clone(), e.version_ms))
889 .collect();
890 for (id, at) in &tombs {
891 self.check_current(generation)?;
892 self.local.remove(id)?;
893 let (id, at) = (id.clone(), *at);
894 self.store.mutate(|l| {
895 l.bury(&id, at);
896 l.pending_tombstones.remove(&id);
897 })?;
898 }
899 rep.tombstones_applied = tombs.len();
900 self.check_current(generation)?;
901
902 let mut pending: Vec<String> = Vec::new();
903
904 match self.flush_tombstones(&canonical, generation, rep) {
906 Ok(true) => {}
907 Ok(false) => pending.push("tombstones pending".into()),
908 Err(e) => return Err(e),
909 }
910
911 struct Upload<I> {
917 header: ItemHeader,
918 item: I,
919 base: Option<RawRecord>,
920 is_new: bool,
921 }
922 let infer = self.config.infer_local_deletions;
923 let mut gone_candidates: HashSet<String> = if infer {
924 let l = self.store.ledger();
925 l.remote_versions
926 .keys()
927 .filter(|id| !l.tombstones.contains_key(*id))
928 .filter(|id| !l.pending_apply.contains_key(*id))
929 .cloned()
930 .collect()
931 } else {
932 HashSet::new()
933 };
934 let mut queued_unseen: HashSet<String> = self
935 .store
936 .ledger()
937 .pending_uploads
938 .iter()
939 .cloned()
940 .collect();
941 let mut matched: HashMap<String, ItemHeader> = HashMap::new();
942 let mut uploads: Vec<Upload<M::Item>> = Vec::new();
943 let mut failed_uploads = 0usize;
944 self.scan_local(|item| {
945 let header = self.mapper.header(&item);
946 if infer {
947 gone_candidates.remove(&header.id);
948 }
949 queued_unseen.remove(&header.id);
950 let canon = canonical.get(&header.id);
951 if let Some(RemoteEntry {
952 state: RemoteState::Live(_),
953 ..
954 }) = canon
955 {
956 matched
957 .entry(header.id.clone())
958 .or_insert_with(|| header.clone());
959 }
960 if self.store.ledger().tombstones.contains_key(&header.id) {
961 rep.blocked_reuploads += 1;
962 return Ok(());
963 }
964 if self.store.ledger().withheld.contains(&header.id) {
965 return Ok(());
966 }
967 let queued = self.store.ledger().pending_uploads.contains(&header.id);
968 match canon {
969 Some(RemoteEntry {
970 state: RemoteState::Live(rh),
971 record,
972 ..
973 }) => {
974 if let Ok(remote_item) = self.mapper.decode(record) {
975 if self
976 .policy
977 .upload_local_over_remote(&header, &item, rh, &remote_item)
978 {
979 uploads.push(Upload {
980 header,
981 item,
982 base: Some(record.clone()),
983 is_new: false,
984 });
985 }
986 }
987 }
988 Some(_) => {}
989 None => {
990 if let Some(known) =
991 self.store.ledger().remote_versions.get(&header.id).cloned()
992 {
993 if known.updated_at_ms < header.updated_at_ms || queued {
995 match self.backend.fetch_record(&known.record_name) {
996 Ok(base) => uploads.push(Upload {
997 header,
998 item,
999 base,
1000 is_new: false,
1001 }),
1002 Err(e) => {
1003 failed_uploads += 1;
1004 self.observer.event("record_fetch_failed", &e.to_string());
1005 }
1006 }
1007 }
1008 } else {
1009 uploads.push(Upload {
1010 header,
1011 item,
1012 base: None,
1013 is_new: true,
1014 });
1015 }
1016 }
1017 }
1018 Ok(())
1019 })?;
1020
1021 if infer {
1023 let gone: Vec<String> = gone_candidates.into_iter().collect();
1024 if !gone.is_empty() {
1025 let now = self.clock.now_ms();
1026 self.store.mutate(|l| {
1027 for id in gone {
1028 l.bury(&id, now);
1029 l.pending_tombstones.insert(id);
1030 }
1031 })?;
1032 if !self.flush_tombstones(&canonical, generation, rep)? {
1033 pending.push("tombstones pending".into());
1034 }
1035 }
1036 }
1037
1038 let stale: Vec<String> = queued_unseen
1041 .into_iter()
1042 .filter(|id| self.store.ledger().pending_uploads.contains(id))
1043 .collect();
1044 if !stale.is_empty() {
1045 self.store.mutate(|l| {
1046 for id in stale {
1047 l.pending_uploads.remove(&id);
1048 }
1049 })?;
1050 }
1051 uploads.sort_by(|a, b| {
1053 (
1054 a.header.created_at_ms,
1055 self.mapper.record_name(&a.header.id),
1056 )
1057 .cmp(&(
1058 b.header.created_at_ms,
1059 self.mapper.record_name(&b.header.id),
1060 ))
1061 });
1062 let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
1063 let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
1064 let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
1065 if self.config.ordered_new_publication {
1066 chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
1067 let mut it = existing.into_iter().peekable();
1068 while it.peek().is_some() {
1069 chunks.push((false, it.by_ref().take(size).collect()));
1070 }
1071 } else {
1072 let mut it = new_ones.into_iter().chain(existing).peekable();
1073 while it.peek().is_some() {
1074 chunks.push((false, it.by_ref().take(size).collect()));
1075 }
1076 }
1077
1078 let mut blocked_new = false;
1079 for (ordered_new, chunk) in chunks {
1080 if ordered_new && blocked_new {
1081 continue;
1082 }
1083 self.check_current(generation)?;
1084 let mut records = Vec::with_capacity(chunk.len());
1085 let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
1086 for u in &chunk {
1087 let name = self.mapper.record_name(&u.header.id);
1088 let mut rec = u
1089 .base
1090 .clone()
1091 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
1092 rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
1093 rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
1094 rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
1095 rec.fields.remove(field::DELETED_AT);
1096 self.mapper.encode(&u.item, &mut rec);
1097 by_name.insert(
1098 rec.record_name.clone(),
1099 (u.header.id.clone(), u.header.updated_at_ms),
1100 );
1101 records.push(rec);
1102 }
1103 let outcomes = self
1106 .backend
1107 .save_records(&records, SavePolicy::IfServerUnchanged)?;
1108 let mut saved: Vec<(String, String, i64)> = Vec::new();
1109 let mut chunk_failed = false;
1110 for o in outcomes {
1111 let Some((id, version)) = by_name.get(&o.record_name) else {
1112 continue;
1113 };
1114 match o.result {
1115 Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
1116 Err(SyncError::Conflict) => {
1117 let upload = chunk
1118 .iter()
1119 .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
1120 .expect("outcome names an upload of this chunk");
1121 let (header, item) = (upload.header.clone(), upload.item.clone());
1122 match self.resolve_conflict(
1123 &header,
1124 &item,
1125 &o.record_name,
1126 o.server_record,
1127 generation,
1128 rep,
1129 )? {
1130 Resolved::Saved => {
1131 saved.push((id.clone(), o.record_name.clone(), *version))
1132 }
1133 Resolved::RemoteWon => {}
1134 Resolved::Failed(why) => {
1135 chunk_failed = true;
1136 failed_uploads += 1;
1137 self.observer.event(
1138 "record_save_failed",
1139 &format!("{}: {why}", o.record_name),
1140 );
1141 }
1142 }
1143 }
1144 Err(e) => {
1145 chunk_failed = true;
1146 failed_uploads += 1;
1147 self.observer
1148 .event("record_save_failed", &format!("{}: {e}", o.record_name));
1149 }
1150 }
1151 }
1152 if ordered_new && chunk_failed {
1153 blocked_new = true;
1154 }
1155 rep.uploaded += saved.len();
1157 self.store.mutate(|l| {
1158 for (id, name, version) in saved {
1159 l.pending_uploads.remove(&id);
1160 l.aliases
1161 .entry(id.clone())
1162 .or_default()
1163 .insert(name.clone());
1164 l.remote_versions.insert(
1165 id,
1166 RemoteVersion {
1167 record_name: name,
1168 updated_at_ms: version,
1169 },
1170 );
1171 }
1172 })?;
1173 }
1174 if failed_uploads > 0 {
1175 pending.push(format!("{failed_uploads} upload(s) pending"));
1176 }
1177
1178 enum Source {
1182 Remote,
1183 Stash(RawRecord),
1184 }
1185 struct Planned {
1186 created: i64,
1187 id: String,
1188 source: Source,
1189 }
1190 let stash_ids: HashSet<String> =
1191 self.store.ledger().pending_apply.keys().cloned().collect();
1192 let mut taken_stash_ids: HashSet<String> = HashSet::new();
1193 let mut plan: Vec<Planned> = Vec::new();
1194 for entry in canonical.values() {
1195 let RemoteState::Live(rh) = &entry.state else {
1196 continue;
1197 };
1198 if self.store.ledger().tombstones.contains_key(&entry.id) {
1199 continue;
1200 }
1201 let take = match matched.get(&entry.id) {
1202 None => true,
1203 Some(lh) => self.policy.accept_remote_over_local(lh, rh),
1204 };
1205 if take {
1206 if stash_ids.contains(&entry.id) {
1207 taken_stash_ids.insert(entry.id.clone());
1208 }
1209 plan.push(Planned {
1210 created: rh.created_at_ms,
1211 id: entry.id.clone(),
1212 source: Source::Remote,
1213 });
1214 }
1215 }
1216 drop(matched);
1217 let stashed: Vec<(String, RawRecord)> = self
1219 .store
1220 .ledger()
1221 .pending_apply
1222 .iter()
1223 .map(|(k, v)| (k.clone(), v.clone()))
1224 .collect();
1225 for (id, record) in stashed {
1226 if taken_stash_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1227 continue;
1228 }
1229 if let Ok(item) = self.mapper.decode(&record) {
1230 let h = self.mapper.header(&item);
1231 plan.push(Planned {
1232 created: h.created_at_ms,
1233 id: h.id,
1234 source: Source::Stash(record),
1235 });
1236 }
1237 }
1238 plan.sort_by(|a, b| (a.created, a.id.as_str()).cmp(&(b.created, b.id.as_str())));
1239 let mut deferred = 0usize;
1240 for chunk in plan.chunks(APPLY_CHUNK) {
1241 self.check_current(generation)?;
1242 let mut items: Vec<M::Item> = Vec::with_capacity(chunk.len());
1243 let mut decoded_at: Vec<usize> = Vec::with_capacity(chunk.len());
1244 for (i, p) in chunk.iter().enumerate() {
1245 let decoded = match &p.source {
1246 Source::Remote => canonical.get(&p.id).map(|e| self.mapper.decode(&e.record)),
1247 Source::Stash(rec) => Some(self.mapper.decode(rec)),
1248 };
1249 if let Some(Ok(item)) = decoded {
1250 items.push(item);
1251 decoded_at.push(i);
1252 }
1253 }
1254 if items.is_empty() {
1255 continue;
1256 }
1257 let outcomes = self.local.apply_remote(&items)?;
1258 if outcomes.len() != items.len() {
1259 return Err(SyncError::Persistence(
1260 "apply_remote returned wrong outcome count".into(),
1261 ));
1262 }
1263 drop(items);
1264 let mut applied_ids: Vec<String> = Vec::new();
1265 let mut stash: Vec<(String, RawRecord)> = Vec::new();
1266 for (i, out) in decoded_at.into_iter().zip(outcomes) {
1267 let p = &chunk[i];
1268 match out {
1269 ApplyOutcome::Applied => {
1270 rep.downloaded += 1;
1271 applied_ids.push(p.id.clone());
1272 }
1273 ApplyOutcome::Deferred(why) => {
1274 deferred += 1;
1275 self.observer
1276 .event("apply_deferred", &format!("{}: {why}", p.id));
1277 let record = match &p.source {
1278 Source::Remote => canonical.get(&p.id).map(|e| e.record.clone()),
1279 Source::Stash(rec) => Some(rec.clone()),
1280 };
1281 if let Some(record) = record {
1282 stash.push((p.id.clone(), record));
1283 }
1284 }
1285 }
1286 }
1287 let touches_ledger = !stash.is_empty()
1288 || applied_ids
1289 .iter()
1290 .any(|id| self.store.ledger().pending_apply.contains_key(id));
1291 if touches_ledger {
1292 self.store.mutate(|l| {
1293 for id in applied_ids {
1294 l.pending_apply.remove(&id);
1295 }
1296 for (id, rec) in stash {
1297 l.pending_apply.insert(id, rec);
1298 }
1299 })?;
1300 }
1301 }
1302 drop(plan);
1303 if deferred > 0 {
1304 pending.push(format!("{deferred} download(s) deferred"));
1305 }
1306
1307 self.check_current(generation)?;
1312 let flags = std::mem::take(&mut self.replay_flags);
1313 let token = fetched.token;
1314 self.store.mutate(|l| {
1315 for e in canonical.values() {
1316 let keep = l
1318 .remote_versions
1319 .get(&e.id)
1320 .is_some_and(|k| k.updated_at_ms > e.version_ms);
1321 if !keep {
1322 l.remote_versions.insert(
1323 e.id.clone(),
1324 RemoteVersion {
1325 record_name: e.record_name.clone(),
1326 updated_at_ms: e.version_ms,
1327 },
1328 );
1329 }
1330 }
1331 if token.is_some() {
1332 l.change_token = token;
1333 }
1334 l.has_synced = true;
1335 for f in flags {
1336 l.flags.insert(f);
1337 }
1338 })?;
1339 if rep.blocked_reuploads > 0 {
1340 self.observer
1341 .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1342 }
1343 Ok(if pending.is_empty() {
1344 None
1345 } else {
1346 Some(pending.join("; "))
1347 })
1348 }
1349
1350 fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1353 let mut reset_token = false;
1354 let mut recreated = false;
1355 'restart: loop {
1356 let mut token = self.store.ledger().change_token.clone();
1357 let mut out = Fetched {
1358 records: Vec::new(),
1359 deleted: Vec::new(),
1360 token: None,
1361 };
1362 loop {
1363 self.check_current(generation)?;
1364 match self.backend.fetch_changes(token.as_deref()) {
1365 Ok(page) => {
1366 out.records.extend(page.records);
1367 out.deleted.extend(page.deleted_record_names);
1368 if page.new_token.is_some() {
1369 token = page.new_token.clone();
1370 out.token = page.new_token;
1371 }
1372 if !page.more_coming {
1373 return Ok(out);
1374 }
1375 }
1376 Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1377 reset_token = true;
1378 self.observer
1379 .event("token_reset", "change token expired; full replay");
1380 self.store.mutate(|l| l.change_token = None)?;
1381 continue 'restart;
1382 }
1383 Err(SyncError::ZoneMissing) if !recreated => {
1384 recreated = true;
1385 self.observer
1386 .event("zone_recreated", "zone missing; re-preparing");
1387 self.store.mutate(|l| {
1391 l.change_token = None;
1392 l.remote_versions.clear();
1393 l.aliases.clear();
1394 })?;
1395 self.backend.prepare()?;
1396 continue 'restart;
1397 }
1398 Err(e) => return Err(e),
1399 }
1400 }
1401 }
1402 }
1403
1404 fn flush_tombstones(
1407 &mut self,
1408 canonical: &BTreeMap<String, RemoteEntry>,
1409 generation: u64,
1410 rep: &mut PassReport,
1411 ) -> SyncResult<bool> {
1412 let ids: Vec<String> = self
1413 .store
1414 .ledger()
1415 .pending_tombstones
1416 .iter()
1417 .cloned()
1418 .collect();
1419 if ids.is_empty() {
1420 return Ok(true);
1421 }
1422 struct Write {
1423 id: String,
1424 record: RawRecord,
1425 }
1426 let mut writes: Vec<Write> = Vec::new();
1427 let mut never_remote: Vec<String> = Vec::new();
1428 for id in &ids {
1429 let mut names: BTreeSet<String> = self
1430 .store
1431 .ledger()
1432 .aliases
1433 .get(id)
1434 .cloned()
1435 .unwrap_or_default();
1436 let known =
1437 self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1438 if names.is_empty() && !known {
1439 never_remote.push(id.clone());
1440 continue;
1441 }
1442 names.insert(self.mapper.record_name(id));
1443 let at = self
1444 .store
1445 .ledger()
1446 .tombstones
1447 .get(id)
1448 .copied()
1449 .unwrap_or_else(|| self.clock.now_ms());
1450 for name in names {
1451 let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1452 Some(e) => e.record.clone(),
1453 None => self
1454 .backend
1455 .fetch_record(&name)
1456 .ok()
1457 .flatten()
1458 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1459 };
1460 write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1461 writes.push(Write {
1462 id: id.clone(),
1463 record,
1464 });
1465 }
1466 }
1467 if !never_remote.is_empty() {
1468 self.store.mutate(|l| {
1470 for id in never_remote {
1471 l.pending_tombstones.remove(&id);
1472 }
1473 })?;
1474 }
1475 let mut failed: BTreeSet<String> = BTreeSet::new();
1476 let size = self
1477 .config
1478 .tombstone_batch
1479 .min(self.backend.max_batch())
1480 .max(1);
1481 for chunk in writes.chunks(size) {
1482 self.check_current(generation)?;
1483 let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1484 let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1485 for w in chunk {
1486 let ok = outcomes
1487 .iter()
1488 .find(|o| o.record_name == w.record.record_name)
1489 .is_some_and(|o| o.result.is_ok());
1490 if !ok {
1491 failed.insert(w.id.clone());
1492 self.observer
1493 .event("tombstone_save_failed", &w.record.record_name);
1494 }
1495 }
1496 }
1497 let acked: Vec<String> = writes
1498 .iter()
1499 .map(|w| w.id.clone())
1500 .collect::<BTreeSet<_>>()
1501 .into_iter()
1502 .filter(|id| !failed.contains(id))
1503 .collect();
1504 rep.tombstones_written += acked.len();
1505 self.store.mutate(|l| {
1506 for id in &acked {
1507 l.pending_tombstones.remove(id);
1508 }
1509 })?;
1510 Ok(failed.is_empty())
1511 }
1512
1513 fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1515 for chunk in records.chunks(self.backend.max_batch().max(1)) {
1516 for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1517 o.result?;
1518 }
1519 }
1520 Ok(())
1521 }
1522}