1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{ConsentRequest, LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17 pub batch_size: usize,
19 pub tombstone_batch: usize,
21 pub ordered_new_publication: bool,
25 pub infer_local_deletions: bool,
29 pub require_consent_on_scope_change: bool,
35 pub backoff: BackoffPolicy,
36}
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub enum ConsentChoice {
41 Merge,
43 StartFresh,
47}
48
49impl Default for SyncConfig {
50 fn default() -> Self {
51 Self {
52 batch_size: 200,
53 tombstone_batch: 50,
54 ordered_new_publication: true,
55 infer_local_deletions: false,
56 require_consent_on_scope_change: false,
57 backoff: BackoffPolicy::default(),
58 }
59 }
60}
61
62#[derive(Debug, Clone, PartialEq)]
63pub enum PassOutcome {
64 Synced,
66 Pending(String),
68 Deferred {
70 secs: f64,
71 },
72 Unavailable(String),
73 Disabled,
74 ConsentRequired(ConsentRequest),
77 Cancelled,
79 Failed(String),
80}
81
82#[derive(Debug, Clone, PartialEq)]
83pub struct PassReport {
84 pub outcome: PassOutcome,
85 pub uploaded: usize,
86 pub downloaded: usize,
87 pub tombstones_applied: usize,
88 pub tombstones_written: usize,
89 pub quarantined: usize,
90 pub blocked_reuploads: usize,
91 pub retry_after_secs: Option<f64>,
93}
94
95impl PassReport {
96 fn new() -> Self {
97 Self {
98 outcome: PassOutcome::Synced,
99 uploaded: 0,
100 downloaded: 0,
101 tombstones_applied: 0,
102 tombstones_written: 0,
103 quarantined: 0,
104 blocked_reuploads: 0,
105 retry_after_secs: None,
106 }
107 }
108}
109
110#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
114pub struct SyncMetrics {
115 pub passes: u64,
117 pub failed_passes: u64,
119 pub last_pass_ms: u64,
121 pub downloaded_total: u64,
122 pub uploaded_total: u64,
123 pub tombstones_applied_total: u64,
124 pub tombstones_written_total: u64,
125 pub remote_versions: u64,
127 pub tombstones: u64,
129 pub pending_uploads: u64,
130 pub pending_tombstones: u64,
131 pub pending_apply: u64,
133}
134
135#[derive(Debug, Clone, Copy, PartialEq)]
137pub struct SyncStatus {
138 pub metrics: SyncMetrics,
140 #[cfg(feature = "process-usage")]
144 pub process: Option<rightkit_child_process::ProcessUsage>,
145}
146
147#[derive(Debug, Default)]
148struct Counters {
149 passes: u64,
150 failed_passes: u64,
151 last_pass_ms: u64,
152 downloaded: u64,
153 uploaded: u64,
154 tombstones_applied: u64,
155 tombstones_written: u64,
156}
157
158const MAX_CONFLICT_ROUNDS: usize = 4;
159const LOCAL_PAGE_SIZE: usize = 1000;
161const APPLY_CHUNK: usize = 1000;
163
164enum Resolved {
165 Saved,
167 RemoteWon,
169 Failed(String),
170}
171
172enum RemoteState {
173 Live(ItemHeader),
176 Tombstone,
177}
178
179struct RemoteEntry {
180 id: String,
181 record_name: String,
182 version_ms: i64,
184 state: RemoteState,
185 record: RawRecord,
186}
187
188impl RemoteEntry {
189 fn is_tombstone(&self) -> bool {
190 matches!(self.state, RemoteState::Tombstone)
191 }
192 fn rank(&self) -> (bool, i64, &str) {
195 (
196 self.is_tombstone(),
197 self.version_ms,
198 self.record_name.as_str(),
199 )
200 }
201}
202
203pub struct SyncEngine<M: RecordMapper> {
204 mapper: Arc<M>,
205 backend: Arc<dyn Backend>,
206 auth: Arc<dyn AuthProvider>,
207 local: Arc<dyn LocalStore<M::Item>>,
208 policy: Box<dyn ConflictPolicy<M::Item>>,
209 clock: Arc<dyn Clock>,
210 observer: Arc<dyn SyncObserver>,
211 hooks: Arc<dyn PassHooks>,
212 config: SyncConfig,
213 store: LedgerStore,
214 replay_flags: Vec<String>,
215 counters: Counters,
216 gate: Option<ConsentRequest>,
218 local_page_size: usize,
219 max_uploads: usize,
221}
222
223pub struct SyncEngineBuilder<M: RecordMapper> {
224 mapper: Arc<M>,
225 backend: Arc<dyn Backend>,
226 local: Arc<dyn LocalStore<M::Item>>,
227 storage: Box<dyn LedgerStorage>,
228 auth: Arc<dyn AuthProvider>,
229 policy: Box<dyn ConflictPolicy<M::Item>>,
230 clock: Arc<dyn Clock>,
231 observer: Arc<dyn SyncObserver>,
232 hooks: Arc<dyn PassHooks>,
233 config: SyncConfig,
234 local_page_size: usize,
235 max_uploads: usize,
236}
237
238impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
239 pub fn new(
240 mapper: Arc<M>,
241 backend: Arc<dyn Backend>,
242 local: Arc<dyn LocalStore<M::Item>>,
243 storage: Box<dyn LedgerStorage>,
244 ) -> Self
245 where
246 M::Item: 'static,
247 {
248 Self {
249 mapper,
250 backend,
251 local,
252 storage,
253 auth: Arc::new(AlwaysAvailable),
254 policy: Box::new(LastWriterWins),
255 clock: Arc::new(SystemClock),
256 observer: Arc::new(NullObserver),
257 hooks: Arc::new(NoHooks),
258 config: SyncConfig::default(),
259 local_page_size: LOCAL_PAGE_SIZE,
260 max_uploads: 0,
261 }
262 }
263 pub fn local_page_size(mut self, n: usize) -> Self {
265 self.local_page_size = n.max(1);
266 self
267 }
268 pub fn max_uploads_per_pass(mut self, n: usize) -> Self {
275 self.max_uploads = n;
276 self
277 }
278 pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
279 self.auth = v;
280 self
281 }
282 pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
283 self.policy = v;
284 self
285 }
286 pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
287 self.clock = v;
288 self
289 }
290 pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
291 self.observer = v;
292 self
293 }
294 pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
295 self.hooks = v;
296 self
297 }
298 pub fn config(mut self, v: SyncConfig) -> Self {
299 self.config = v;
300 self
301 }
302 pub fn build(self) -> SyncResult<SyncEngine<M>> {
303 let store = LedgerStore::open(self.storage)?;
304 if store.recovered_from_corruption {
305 self.observer
306 .event("ledger_recovered", "ledger unreadable; full replay");
307 }
308 Ok(SyncEngine {
309 mapper: self.mapper,
310 backend: self.backend,
311 auth: self.auth,
312 local: self.local,
313 policy: self.policy,
314 clock: self.clock,
315 observer: self.observer,
316 hooks: self.hooks,
317 config: self.config,
318 store,
319 replay_flags: Vec::new(),
320 counters: Counters::default(),
321 gate: None,
322 local_page_size: self.local_page_size,
323 max_uploads: self.max_uploads,
324 })
325 }
326}
327
328struct Fetched {
329 records: Vec<RawRecord>,
330 deleted: Vec<String>,
331 token: Option<Vec<u8>>,
332}
333
334impl<M: RecordMapper + 'static> SyncEngine<M>
335where
336 M::Item: 'static,
337{
338 pub fn ledger(&self) -> &crate::ledger::Ledger {
339 self.store.ledger()
340 }
341
342 pub fn consent_request(&self) -> Option<ConsentRequest> {
345 self.store.ledger().pending_consent.clone()
346 }
347
348 pub fn grant_consent(&mut self, to: &str, choice: ConsentChoice) -> SyncResult<()> {
354 let Some(req) = self.store.ledger().pending_consent.clone() else {
355 return Err(SyncError::Transient(
356 "no scope change is awaiting consent".into(),
357 ));
358 };
359 if req.to != to {
360 return Err(SyncError::Transient(
361 "consent names a different account than the one pending".into(),
362 ));
363 }
364 if let Some(current) = self.auth.account_scope() {
365 if current != req.to {
366 return Err(SyncError::Transient(
367 "the signed-in account changed again; consent is stale".into(),
368 ));
369 }
370 }
371 let withheld: BTreeSet<String> = match choice {
372 ConsentChoice::Merge => BTreeSet::new(),
373 ConsentChoice::StartFresh => {
374 let mut ids = BTreeSet::new();
375 self.scan_local(|i| {
376 ids.insert(self.mapper.header(&i).id);
377 Ok(())
378 })?;
379 ids
380 }
381 };
382 self.hooks.on_account_switch(&req.from, &req.to);
383 let scope = req.to.clone();
384 self.store.mutate(|l| {
385 *l = crate::ledger::Ledger::default();
386 l.scope = Some(scope);
387 l.withheld = withheld;
388 })?;
389 self.gate = None;
390 self.observer.event(
391 "consent_granted",
392 match choice {
393 ConsentChoice::Merge => "merge",
394 ConsentChoice::StartFresh => "start_fresh",
395 },
396 );
397 Ok(())
398 }
399
400 fn scan_local(&self, mut each: impl FnMut(M::Item) -> SyncResult<()>) -> SyncResult<()> {
404 let size = self.local_page_size.max(1);
405 let mut cursor: Option<String> = None;
406 loop {
407 let (page, last) = match self.local.list_page(cursor.as_deref(), size)? {
408 Some(p) => {
409 let last = p.len() < size;
410 (p, last)
411 }
412 None => (self.local.all()?, true),
413 };
414 let next = if last {
415 None
416 } else {
417 page.last().map(|i| self.mapper.header(i).id)
418 };
419 if let (Some(n), Some(c)) = (next.as_ref(), cursor.as_ref()) {
420 if n <= c {
421 return Err(SyncError::Persistence(
422 "LocalStore::list_page did not advance past its cursor".into(),
423 ));
424 }
425 }
426 for item in page {
427 each(item)?;
428 }
429 match next {
430 Some(n) => cursor = Some(n),
431 None => return Ok(()),
432 }
433 }
434 }
435
436 pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
438 let id = id.to_string();
439 self.store.mutate(|l| {
440 if !l.tombstones.contains_key(&id) {
441 l.pending_uploads.insert(id);
442 }
443 })
444 }
445
446 pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
448 let now = self.clock.now_ms();
449 let id = id.to_string();
450 self.store.mutate(|l| {
451 l.bury(&id, now);
452 l.pending_tombstones.insert(id);
453 })
454 }
455
456 pub fn note_deleted_many(
469 &mut self,
470 ids: &[String],
471 unconfirmed: &BTreeSet<String>,
472 ) -> SyncResult<usize> {
473 let now = self.clock.now_ms();
474 let queue: Vec<String> = {
475 let ledger = self.store.ledger();
476 ids.iter()
477 .filter(|id| {
478 let confirmed = ledger.tombstones.contains_key(*id)
479 && !ledger.pending_tombstones.contains(*id);
480 !confirmed || unconfirmed.contains(*id)
481 })
482 .cloned()
483 .collect()
484 };
485 if queue.is_empty() {
486 return Ok(0);
487 }
488 self.store.mutate(|l| {
489 for id in &queue {
490 if !l.tombstones.contains_key(id) {
491 l.bury(id, now);
492 }
493 l.pending_tombstones.insert(id.clone());
494 }
495 })?;
496 Ok(queue.len())
497 }
498
499 pub fn withdraw_pending_tombstones(
511 &mut self,
512 lift: &[String],
513 keep: &[String],
514 ) -> SyncResult<usize> {
515 let (queued, lifted): (BTreeSet<String>, BTreeSet<String>) = {
516 let ledger = self.store.ledger();
517 let kept: BTreeSet<&String> = keep.iter().collect();
518 let queued: BTreeSet<String> = lift
519 .iter()
520 .chain(keep.iter())
521 .filter(|id| ledger.pending_tombstones.contains(*id))
522 .cloned()
523 .collect();
524 let lifted: BTreeSet<String> = lift
525 .iter()
526 .filter(|id| queued.contains(*id) && !kept.contains(id))
527 .cloned()
528 .collect();
529 (queued, lifted)
530 };
531 if queued.is_empty() {
532 return Ok(0);
533 }
534 self.store.mutate(|l| {
535 for id in &queued {
536 l.pending_tombstones.remove(id);
537 }
538 for id in &lifted {
539 l.tombstones.remove(id);
540 }
541 })?;
542 Ok(queued.len())
543 }
544
545 pub fn request_full_replay_once(&mut self, flag: &str) {
547 if !self.store.ledger().flags.contains(flag) {
548 self.replay_flags.push(flag.to_string());
549 }
550 }
551
552 pub fn run_pass(&mut self) -> PassReport {
554 let started = Instant::now();
555 let rep = self.run_pass_once();
556 self.record_pass(&rep, started.elapsed());
557 rep
558 }
559
560 fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
561 let c = &mut self.counters;
562 c.passes += 1;
563 if matches!(
564 rep.outcome,
565 PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
566 ) {
567 c.failed_passes += 1;
568 }
569 c.last_pass_ms = elapsed.as_millis() as u64;
570 c.downloaded += rep.downloaded as u64;
571 c.uploaded += rep.uploaded as u64;
572 c.tombstones_applied += rep.tombstones_applied as u64;
573 c.tombstones_written += rep.tombstones_written as u64;
574 }
575
576 pub fn status(&self) -> SyncStatus {
579 SyncStatus {
580 metrics: self.metrics(),
581 #[cfg(feature = "process-usage")]
582 process: rightkit_child_process::process_usage().ok(),
583 }
584 }
585
586 pub fn metrics(&self) -> SyncMetrics {
588 let l = self.store.ledger();
589 let c = &self.counters;
590 SyncMetrics {
591 passes: c.passes,
592 failed_passes: c.failed_passes,
593 last_pass_ms: c.last_pass_ms,
594 downloaded_total: c.downloaded,
595 uploaded_total: c.uploaded,
596 tombstones_applied_total: c.tombstones_applied,
597 tombstones_written_total: c.tombstones_written,
598 remote_versions: l.remote_versions.len() as u64,
599 tombstones: l.tombstones.len() as u64,
600 pending_uploads: l.pending_uploads.len() as u64,
601 pending_tombstones: l.pending_tombstones.len() as u64,
602 pending_apply: l.pending_apply.len() as u64,
603 }
604 }
605
606 fn run_pass_once(&mut self) -> PassReport {
607 let mut rep = PassReport::new();
608 if !self.auth.enabled() {
609 rep.outcome = PassOutcome::Disabled;
610 return rep;
611 }
612 let now = self.clock.now_ms();
613 if let Some(until) = self.store.ledger().retry_after_ms {
614 if until > now {
615 let secs = (until - now) as f64 / 1000.0;
616 rep.outcome = PassOutcome::Deferred { secs };
617 rep.retry_after_secs = Some(secs);
618 return rep;
619 }
620 }
621 let generation = self.auth.generation();
622 self.gate = None;
623 let result = self.pass_inner(generation, &mut rep);
624 if let Some(req) = self.gate.take() {
625 if result.is_ok() {
626 rep.outcome = PassOutcome::ConsentRequired(req);
627 return rep;
628 }
629 }
630 match result {
631 Ok(pending) => {
632 let result = self.store.mutate(|l| {
633 if pending.is_none() {
634 l.consecutive_failures = 0;
635 l.retry_after_ms = None;
636 }
637 });
638 match (result, pending) {
639 (Err(e), _) => self.fail(e, &mut rep),
640 (Ok(()), None) => rep.outcome = PassOutcome::Synced,
641 (Ok(()), Some(why)) => {
642 let secs = self.config.backoff.pending_floor_secs;
643 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
644 let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
645 rep.retry_after_secs = Some(secs);
646 rep.outcome = PassOutcome::Pending(why);
647 }
648 }
649 }
650 Err(e) => self.fail(e, &mut rep),
651 }
652 rep
653 }
654
655 fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
656 if e == SyncError::Cancelled {
657 rep.outcome = PassOutcome::Cancelled;
658 return;
659 }
660 self.observer.event("pass_failed", &e.to_string());
661 let failures = self.store.ledger().consecutive_failures.saturating_add(1);
662 let secs = self
663 .config
664 .backoff
665 .delay_secs(failures, e.retry_hint_secs());
666 let until = self.clock.now_ms() + (secs * 1000.0) as i64;
667 let _ = self.store.mutate(|l| {
668 l.consecutive_failures = failures;
669 l.retry_after_ms = Some(until);
670 });
671 rep.retry_after_secs = Some(secs);
672 rep.outcome = match e {
673 SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
674 other => PassOutcome::Failed(other.to_string()),
675 };
676 }
677
678 fn resolve_conflict(
683 &mut self,
684 header: &ItemHeader,
685 item: &M::Item,
686 name: &str,
687 mut server: Option<RawRecord>,
688 generation: u64,
689 rep: &mut PassReport,
690 ) -> SyncResult<Resolved> {
691 for _ in 0..MAX_CONFLICT_ROUNDS {
692 self.check_current(generation)?;
693 let server_rec = match server.take() {
694 Some(r) => r,
695 None => match self.backend.fetch_record(name)? {
696 Some(r) => r,
697 None => return Ok(Resolved::Failed("conflicting record vanished".into())),
698 },
699 };
700 self.observer.event("save_conflict", name);
701 if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
702 self.local.remove(&id)?;
703 self.store.mutate(|l| l.bury(&id, deleted_at))?;
704 rep.tombstones_applied += 1;
705 return Ok(Resolved::RemoteWon);
706 }
707 let remote_item = match self.mapper.decode(&server_rec) {
708 Ok(i) => i,
709 Err(e) => {
710 rep.quarantined += 1;
711 return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
712 }
713 };
714 let rh = self.mapper.header(&remote_item);
715 if self
716 .policy
717 .upload_local_over_remote(header, item, &rh, &remote_item)
718 {
719 let mut rec = server_rec;
720 rec.set(field::ID, FieldValue::Text(header.id.clone()));
721 rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
722 rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
723 rec.fields.remove(field::DELETED_AT);
724 self.mapper.encode(item, &mut rec);
725 let outcomes = self
726 .backend
727 .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
728 let Some(o) = outcomes.into_iter().next() else {
729 return Ok(Resolved::Failed("no save outcome".into()));
730 };
731 match o.result {
732 Ok(()) => return Ok(Resolved::Saved),
733 Err(SyncError::Conflict) => server = o.server_record,
734 Err(e) => return Ok(Resolved::Failed(e.to_string())),
735 }
736 continue;
737 }
738 let version = RemoteVersion {
739 record_name: name.to_string(),
740 updated_at_ms: rh.updated_at_ms,
741 };
742 if self.policy.accept_remote_over_local(header, &rh) {
743 self.check_current(generation)?;
744 match self.local.apply_remote(&[remote_item])?.into_iter().next() {
745 Some(ApplyOutcome::Deferred(_)) => {
746 let id = header.id.clone();
747 self.store.mutate(|l| {
748 l.pending_apply.insert(id, server_rec);
749 })?;
750 }
751 _ => rep.downloaded += 1,
752 }
753 }
754 let id = header.id.clone();
755 self.store.mutate(|l| {
756 l.pending_uploads.remove(&id);
757 l.remote_versions.insert(id, version);
758 })?;
759 return Ok(Resolved::RemoteWon);
760 }
761 Ok(Resolved::Failed("conflict retries exhausted".into()))
762 }
763
764 fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
772 self.check_current(generation)?;
773 let Some(scope) = self.auth.account_scope() else {
774 return Ok(());
775 };
776 match self.store.ledger().scope.clone() {
777 Some(bound) if bound == scope => {
778 if self.store.ledger().pending_consent.is_some() {
779 self.store.mutate(|l| l.pending_consent = None)?;
781 }
782 Ok(())
783 }
784 Some(bound) if self.config.require_consent_on_scope_change => {
785 let req = ConsentRequest {
786 from: bound,
787 to: scope,
788 };
789 if self.store.ledger().pending_consent.as_ref() != Some(&req) {
790 let stored = req.clone();
791 self.store.mutate(|l| l.pending_consent = Some(stored))?;
792 self.observer
793 .event("consent_required", &format!("{} -> {}", req.from, req.to));
794 }
795 self.gate = Some(req);
796 Ok(())
797 }
798 None => {
799 let dropped = {
803 let l = self.store.ledger();
804 l.change_token.is_some()
805 || !l.remote_versions.is_empty()
806 || !l.aliases.is_empty()
807 || !l.pending_tombstones.is_empty()
808 };
809 self.store.mutate(|l| {
810 l.scope = Some(scope);
811 l.change_token = None;
812 l.remote_versions.clear();
813 l.aliases.clear();
814 l.pending_tombstones.clear();
815 l.has_synced = false;
816 })?;
817 if dropped {
818 self.observer
819 .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
820 }
821 Ok(())
822 }
823 Some(bound) => {
824 self.hooks.on_account_switch(&bound, &scope);
825 self.store.mutate(|l| {
826 *l = crate::ledger::Ledger::default();
827 l.scope = Some(scope.clone());
828 })?;
829 self.observer.event("account_switched", "ledger reset");
830 Ok(())
831 }
832 }
833 }
834
835 fn check_current(&self, generation: u64) -> SyncResult<()> {
836 if self.auth.enabled() && self.auth.generation() == generation {
837 Ok(())
838 } else {
839 Err(SyncError::Cancelled)
840 }
841 }
842
843 fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
845 self.auth.availability()?;
846 let generation = self.auth.generation().max(generation);
849 self.bind_account(generation)?;
850 if self.gate.is_some() {
851 return Ok(None);
852 }
853 self.backend.prepare()?;
854 self.check_current(generation)?;
855
856 let mut extras = self.store.ledger().extras.clone();
858 let outbound = self.hooks.outbound(&mut extras);
859 if !outbound.is_empty() {
860 self.check_current(generation)?;
861 self.write_required(&outbound)?;
862 }
863 self.check_current(generation)?;
864 self.store.mutate(|l| l.extras = extras)?;
865
866 if !self.replay_flags.is_empty() {
868 self.store.mutate(|l| l.change_token = None)?;
869 }
870
871 let fetched = self.fetch_all(generation)?;
872 let mapper_type = self.mapper.record_type().to_string();
873 let mut entries: Vec<RemoteEntry> = Vec::new();
874 let mut foreign: Vec<RawRecord> = Vec::new();
875 for record in fetched.records {
876 if record.record_type != mapper_type {
877 foreign.push(record);
878 continue;
879 }
880 if let Some((id, deleted_at)) = read_tombstone(&record) {
881 entries.push(RemoteEntry {
882 id,
883 record_name: record.record_name.clone(),
884 version_ms: deleted_at,
885 state: RemoteState::Tombstone,
886 record,
887 });
888 continue;
889 }
890 match self.mapper.decode(&record) {
891 Ok(item) => {
892 let h = self.mapper.header(&item);
893 entries.push(RemoteEntry {
894 id: h.id.clone(),
895 record_name: record.record_name.clone(),
896 version_ms: h.updated_at_ms,
897 state: RemoteState::Live(h),
898 record,
899 });
900 }
901 Err(why) => {
902 rep.quarantined += 1;
904 self.observer.event(
905 "record_quarantined",
906 &format!("{}: {why}", record.record_name),
907 );
908 }
909 }
910 }
911 for name in &fetched.deleted {
913 let known = self
914 .store
915 .ledger()
916 .aliases
917 .iter()
918 .find(|(_, names)| names.contains(name))
919 .map(|(id, _)| id.clone())
920 .or_else(|| {
921 self.store
922 .ledger()
923 .remote_versions
924 .contains_key(name)
925 .then(|| name.clone())
926 });
927 if let Some(id) = known {
928 if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
929 let mut record = RawRecord::new(&mapper_type, name);
930 let now = self.clock.now_ms();
931 write_tombstone(&mut record, &id, now, &[]);
932 entries.push(RemoteEntry {
933 id,
934 record_name: name.clone(),
935 version_ms: now,
936 state: RemoteState::Tombstone,
937 record,
938 });
939 }
940 }
941 }
942 if rep.quarantined > 0 {
943 let n = rep.quarantined as u64;
944 self.store.mutate(|l| l.quarantined_total += n)?;
945 }
946
947 self.store.mutate(|l| {
949 for e in &entries {
950 l.aliases
951 .entry(e.id.clone())
952 .or_default()
953 .insert(e.record_name.clone());
954 }
955 })?;
956
957 let mut canonical: BTreeMap<String, RemoteEntry> = BTreeMap::new();
959 for entry in entries {
960 match canonical.get(&entry.id) {
961 Some(cur) if cur.rank() >= entry.rank() => {}
962 _ => {
963 canonical.insert(entry.id.clone(), entry);
964 }
965 }
966 }
967
968 if !foreign.is_empty() {
970 let mut extras = self.store.ledger().extras.clone();
971 let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
972 if !responses.is_empty() {
978 self.check_current(generation)?;
979 if let Err(e) = self.write_required(&responses) {
980 self.observer.event("foreign_write_failed", &e.to_string());
981 return Err(e);
982 }
983 }
984 self.check_current(generation)?;
985 self.store.mutate(|l| l.extras = extras)?;
986 }
987
988 let tombs: Vec<(String, i64)> = canonical
990 .values()
991 .filter(|e| e.is_tombstone())
992 .map(|e| (e.id.clone(), e.version_ms))
993 .collect();
994 for (id, at) in &tombs {
995 self.check_current(generation)?;
996 self.local.remove(id)?;
997 let (id, at) = (id.clone(), *at);
998 self.store.mutate(|l| {
999 l.bury(&id, at);
1000 l.pending_tombstones.remove(&id);
1001 })?;
1002 }
1003 rep.tombstones_applied = tombs.len();
1004 self.check_current(generation)?;
1005
1006 let mut pending: Vec<String> = Vec::new();
1007
1008 match self.flush_tombstones(&canonical, generation, rep) {
1010 Ok(true) => {}
1011 Ok(false) => pending.push("tombstones pending".into()),
1012 Err(e) => return Err(e),
1013 }
1014
1015 struct Upload<I> {
1021 header: ItemHeader,
1022 item: I,
1023 base: Option<RawRecord>,
1024 is_new: bool,
1025 }
1026 struct Queued<I> {
1029 key: (i64, String),
1030 upload: Upload<I>,
1031 }
1032 impl<I> PartialEq for Queued<I> {
1033 fn eq(&self, other: &Self) -> bool {
1034 self.key == other.key
1035 }
1036 }
1037 impl<I> Eq for Queued<I> {}
1038 impl<I> PartialOrd for Queued<I> {
1039 fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
1040 Some(self.cmp(other))
1041 }
1042 }
1043 impl<I> Ord for Queued<I> {
1044 fn cmp(&self, other: &Self) -> std::cmp::Ordering {
1045 self.key.cmp(&other.key)
1046 }
1047 }
1048 struct UploadQueue<I> {
1051 cap: usize,
1052 heap: std::collections::BinaryHeap<Queued<I>>,
1053 held_back: usize,
1054 }
1055 impl<I> UploadQueue<I> {
1056 fn offer(&mut self, key: (i64, String), upload: Upload<I>) {
1057 if self.cap == 0 || self.heap.len() < self.cap {
1058 self.heap.push(Queued { key, upload });
1059 return;
1060 }
1061 self.held_back += 1;
1062 let replaces = self.heap.peek().is_some_and(|top| key < top.key);
1063 if replaces {
1064 self.heap.pop();
1065 self.heap.push(Queued { key, upload });
1066 }
1067 }
1068 }
1069 let infer = self.config.infer_local_deletions;
1070 let mut gone_candidates: HashSet<String> = if infer {
1071 let l = self.store.ledger();
1072 l.remote_versions
1073 .keys()
1074 .filter(|id| !l.tombstones.contains_key(*id))
1075 .filter(|id| !l.pending_apply.contains_key(*id))
1076 .cloned()
1077 .collect()
1078 } else {
1079 HashSet::new()
1080 };
1081 let mut queued_unseen: HashSet<String> = self
1082 .store
1083 .ledger()
1084 .pending_uploads
1085 .iter()
1086 .cloned()
1087 .collect();
1088 let mut matched: HashMap<String, ItemHeader> = HashMap::new();
1089 let mut queue: UploadQueue<M::Item> = UploadQueue {
1090 cap: self.max_uploads,
1091 heap: std::collections::BinaryHeap::new(),
1092 held_back: 0,
1093 };
1094 let mut failed_uploads = 0usize;
1095 self.scan_local(|item| {
1096 let header = self.mapper.header(&item);
1097 if infer {
1098 gone_candidates.remove(&header.id);
1099 }
1100 queued_unseen.remove(&header.id);
1101 let canon = canonical.get(&header.id);
1102 if let Some(RemoteEntry {
1103 state: RemoteState::Live(_),
1104 ..
1105 }) = canon
1106 {
1107 matched
1108 .entry(header.id.clone())
1109 .or_insert_with(|| header.clone());
1110 }
1111 if self.store.ledger().tombstones.contains_key(&header.id) {
1112 rep.blocked_reuploads += 1;
1113 return Ok(());
1114 }
1115 if self.store.ledger().withheld.contains(&header.id) {
1116 return Ok(());
1117 }
1118 let queued = self.store.ledger().pending_uploads.contains(&header.id);
1119 match canon {
1120 Some(RemoteEntry {
1121 state: RemoteState::Live(rh),
1122 record,
1123 ..
1124 }) => {
1125 if let Ok(remote_item) = self.mapper.decode(record) {
1126 if self
1127 .policy
1128 .upload_local_over_remote(&header, &item, rh, &remote_item)
1129 {
1130 let key = (header.created_at_ms, self.mapper.record_name(&header.id));
1131 queue.offer(
1132 key,
1133 Upload {
1134 header,
1135 item,
1136 base: Some(record.clone()),
1137 is_new: false,
1138 },
1139 );
1140 }
1141 }
1142 }
1143 Some(_) => {}
1144 None => {
1145 if let Some(known) =
1146 self.store.ledger().remote_versions.get(&header.id).cloned()
1147 {
1148 if known.updated_at_ms < header.updated_at_ms || queued {
1150 match self.backend.fetch_record(&known.record_name) {
1151 Ok(base) => {
1152 let key =
1153 (header.created_at_ms, self.mapper.record_name(&header.id));
1154 queue.offer(
1155 key,
1156 Upload {
1157 header,
1158 item,
1159 base,
1160 is_new: false,
1161 },
1162 );
1163 }
1164 Err(e) => {
1165 failed_uploads += 1;
1166 self.observer.event("record_fetch_failed", &e.to_string());
1167 }
1168 }
1169 }
1170 } else {
1171 let key = (header.created_at_ms, self.mapper.record_name(&header.id));
1172 queue.offer(
1173 key,
1174 Upload {
1175 header,
1176 item,
1177 base: None,
1178 is_new: true,
1179 },
1180 );
1181 }
1182 }
1183 }
1184 Ok(())
1185 })?;
1186 let held_back_uploads = queue.held_back;
1187 let mut uploads: Vec<Upload<M::Item>> =
1188 queue.heap.into_iter().map(|queued| queued.upload).collect();
1189
1190 if infer {
1192 let gone: Vec<String> = gone_candidates.into_iter().collect();
1193 if !gone.is_empty() {
1194 let now = self.clock.now_ms();
1195 self.store.mutate(|l| {
1196 for id in gone {
1197 l.bury(&id, now);
1198 l.pending_tombstones.insert(id);
1199 }
1200 })?;
1201 if !self.flush_tombstones(&canonical, generation, rep)? {
1202 pending.push("tombstones pending".into());
1203 }
1204 }
1205 }
1206
1207 let stale: Vec<String> = queued_unseen
1210 .into_iter()
1211 .filter(|id| self.store.ledger().pending_uploads.contains(id))
1212 .collect();
1213 if !stale.is_empty() {
1214 self.store.mutate(|l| {
1215 for id in stale {
1216 l.pending_uploads.remove(&id);
1217 }
1218 })?;
1219 }
1220 uploads.sort_by(|a, b| {
1222 (
1223 a.header.created_at_ms,
1224 self.mapper.record_name(&a.header.id),
1225 )
1226 .cmp(&(
1227 b.header.created_at_ms,
1228 self.mapper.record_name(&b.header.id),
1229 ))
1230 });
1231 let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
1232 let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
1233 let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
1234 if self.config.ordered_new_publication {
1235 chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
1236 let mut it = existing.into_iter().peekable();
1237 while it.peek().is_some() {
1238 chunks.push((false, it.by_ref().take(size).collect()));
1239 }
1240 } else {
1241 let mut it = new_ones.into_iter().chain(existing).peekable();
1242 while it.peek().is_some() {
1243 chunks.push((false, it.by_ref().take(size).collect()));
1244 }
1245 }
1246
1247 let mut blocked_new = false;
1248 for (ordered_new, chunk) in chunks {
1249 if ordered_new && blocked_new {
1250 continue;
1251 }
1252 self.check_current(generation)?;
1253 let mut records = Vec::with_capacity(chunk.len());
1254 let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
1255 for u in &chunk {
1256 let name = self.mapper.record_name(&u.header.id);
1257 let mut rec = u
1258 .base
1259 .clone()
1260 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
1261 rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
1262 rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
1263 rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
1264 rec.fields.remove(field::DELETED_AT);
1265 self.mapper.encode(&u.item, &mut rec);
1266 by_name.insert(
1267 rec.record_name.clone(),
1268 (u.header.id.clone(), u.header.updated_at_ms),
1269 );
1270 records.push(rec);
1271 }
1272 let outcomes = self
1275 .backend
1276 .save_records(&records, SavePolicy::IfServerUnchanged)?;
1277 let mut saved: Vec<(String, String, i64)> = Vec::new();
1278 let mut chunk_failed = false;
1279 for o in outcomes {
1280 let Some((id, version)) = by_name.get(&o.record_name) else {
1281 continue;
1282 };
1283 match o.result {
1284 Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
1285 Err(SyncError::Conflict) => {
1286 let upload = chunk
1287 .iter()
1288 .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
1289 .expect("outcome names an upload of this chunk");
1290 let (header, item) = (upload.header.clone(), upload.item.clone());
1291 match self.resolve_conflict(
1292 &header,
1293 &item,
1294 &o.record_name,
1295 o.server_record,
1296 generation,
1297 rep,
1298 )? {
1299 Resolved::Saved => {
1300 saved.push((id.clone(), o.record_name.clone(), *version))
1301 }
1302 Resolved::RemoteWon => {}
1303 Resolved::Failed(why) => {
1304 chunk_failed = true;
1305 failed_uploads += 1;
1306 self.observer.event(
1307 "record_save_failed",
1308 &format!("{}: {why}", o.record_name),
1309 );
1310 }
1311 }
1312 }
1313 Err(e) => {
1314 chunk_failed = true;
1315 failed_uploads += 1;
1316 self.observer
1317 .event("record_save_failed", &format!("{}: {e}", o.record_name));
1318 }
1319 }
1320 }
1321 if ordered_new && chunk_failed {
1322 blocked_new = true;
1323 }
1324 rep.uploaded += saved.len();
1326 self.store.mutate(|l| {
1327 for (id, name, version) in saved {
1328 l.pending_uploads.remove(&id);
1329 l.aliases
1330 .entry(id.clone())
1331 .or_default()
1332 .insert(name.clone());
1333 l.remote_versions.insert(
1334 id,
1335 RemoteVersion {
1336 record_name: name,
1337 updated_at_ms: version,
1338 },
1339 );
1340 }
1341 })?;
1342 }
1343 if failed_uploads > 0 {
1344 pending.push(format!("{failed_uploads} upload(s) pending"));
1345 }
1346 if held_back_uploads > 0 {
1347 self.observer
1348 .event("uploads_deferred", &held_back_uploads.to_string());
1349 }
1350
1351 enum Source {
1355 Remote,
1356 Stash(RawRecord),
1357 }
1358 struct Planned {
1359 created: i64,
1360 id: String,
1361 source: Source,
1362 }
1363 let stash_ids: HashSet<String> =
1364 self.store.ledger().pending_apply.keys().cloned().collect();
1365 let mut taken_stash_ids: HashSet<String> = HashSet::new();
1366 let mut plan: Vec<Planned> = Vec::new();
1367 for entry in canonical.values() {
1368 let RemoteState::Live(rh) = &entry.state else {
1369 continue;
1370 };
1371 if self.store.ledger().tombstones.contains_key(&entry.id) {
1372 continue;
1373 }
1374 let take = match matched.get(&entry.id) {
1375 None => true,
1376 Some(lh) => self.policy.accept_remote_over_local(lh, rh),
1377 };
1378 if take {
1379 if stash_ids.contains(&entry.id) {
1380 taken_stash_ids.insert(entry.id.clone());
1381 }
1382 plan.push(Planned {
1383 created: rh.created_at_ms,
1384 id: entry.id.clone(),
1385 source: Source::Remote,
1386 });
1387 }
1388 }
1389 drop(matched);
1390 let stashed: Vec<(String, RawRecord)> = self
1392 .store
1393 .ledger()
1394 .pending_apply
1395 .iter()
1396 .map(|(k, v)| (k.clone(), v.clone()))
1397 .collect();
1398 for (id, record) in stashed {
1399 if taken_stash_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1400 continue;
1401 }
1402 if let Ok(item) = self.mapper.decode(&record) {
1403 let h = self.mapper.header(&item);
1404 plan.push(Planned {
1405 created: h.created_at_ms,
1406 id: h.id,
1407 source: Source::Stash(record),
1408 });
1409 }
1410 }
1411 plan.sort_by(|a, b| (a.created, a.id.as_str()).cmp(&(b.created, b.id.as_str())));
1412 let mut deferred = 0usize;
1413 for chunk in plan.chunks(APPLY_CHUNK) {
1414 self.check_current(generation)?;
1415 let mut items: Vec<M::Item> = Vec::with_capacity(chunk.len());
1416 let mut decoded_at: Vec<usize> = Vec::with_capacity(chunk.len());
1417 for (i, p) in chunk.iter().enumerate() {
1418 let decoded = match &p.source {
1419 Source::Remote => canonical.get(&p.id).map(|e| self.mapper.decode(&e.record)),
1420 Source::Stash(rec) => Some(self.mapper.decode(rec)),
1421 };
1422 if let Some(Ok(item)) = decoded {
1423 items.push(item);
1424 decoded_at.push(i);
1425 }
1426 }
1427 if items.is_empty() {
1428 continue;
1429 }
1430 let outcomes = self.local.apply_remote(&items)?;
1431 if outcomes.len() != items.len() {
1432 return Err(SyncError::Persistence(
1433 "apply_remote returned wrong outcome count".into(),
1434 ));
1435 }
1436 drop(items);
1437 let mut applied_ids: Vec<String> = Vec::new();
1438 let mut stash: Vec<(String, RawRecord)> = Vec::new();
1439 for (i, out) in decoded_at.into_iter().zip(outcomes) {
1440 let p = &chunk[i];
1441 match out {
1442 ApplyOutcome::Applied => {
1443 rep.downloaded += 1;
1444 applied_ids.push(p.id.clone());
1445 }
1446 ApplyOutcome::Deferred(why) => {
1447 deferred += 1;
1448 self.observer
1449 .event("apply_deferred", &format!("{}: {why}", p.id));
1450 let record = match &p.source {
1451 Source::Remote => canonical.get(&p.id).map(|e| e.record.clone()),
1452 Source::Stash(rec) => Some(rec.clone()),
1453 };
1454 if let Some(record) = record {
1455 stash.push((p.id.clone(), record));
1456 }
1457 }
1458 }
1459 }
1460 let touches_ledger = !stash.is_empty()
1461 || applied_ids
1462 .iter()
1463 .any(|id| self.store.ledger().pending_apply.contains_key(id));
1464 if touches_ledger {
1465 self.store.mutate(|l| {
1466 for id in applied_ids {
1467 l.pending_apply.remove(&id);
1468 }
1469 for (id, rec) in stash {
1470 l.pending_apply.insert(id, rec);
1471 }
1472 })?;
1473 }
1474 }
1475 drop(plan);
1476 if deferred > 0 {
1477 pending.push(format!("{deferred} download(s) deferred"));
1478 }
1479
1480 self.check_current(generation)?;
1485 let flags = std::mem::take(&mut self.replay_flags);
1486 let token = fetched.token;
1487 self.store.mutate(|l| {
1488 for e in canonical.values() {
1489 let keep = l
1491 .remote_versions
1492 .get(&e.id)
1493 .is_some_and(|k| k.updated_at_ms > e.version_ms);
1494 if !keep {
1495 l.remote_versions.insert(
1496 e.id.clone(),
1497 RemoteVersion {
1498 record_name: e.record_name.clone(),
1499 updated_at_ms: e.version_ms,
1500 },
1501 );
1502 }
1503 }
1504 if token.is_some() {
1505 l.change_token = token;
1506 }
1507 l.has_synced = true;
1508 for f in flags {
1509 l.flags.insert(f);
1510 }
1511 })?;
1512 if rep.blocked_reuploads > 0 {
1513 self.observer
1514 .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1515 }
1516 Ok(if pending.is_empty() {
1517 None
1518 } else {
1519 Some(pending.join("; "))
1520 })
1521 }
1522
1523 fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1526 let mut reset_token = false;
1527 let mut recreated = false;
1528 'restart: loop {
1529 let mut token = self.store.ledger().change_token.clone();
1530 let mut out = Fetched {
1531 records: Vec::new(),
1532 deleted: Vec::new(),
1533 token: None,
1534 };
1535 loop {
1536 self.check_current(generation)?;
1537 match self.backend.fetch_changes(token.as_deref()) {
1538 Ok(page) => {
1539 out.records.extend(page.records);
1540 out.deleted.extend(page.deleted_record_names);
1541 if page.new_token.is_some() {
1542 token = page.new_token.clone();
1543 out.token = page.new_token;
1544 }
1545 if !page.more_coming {
1546 return Ok(out);
1547 }
1548 }
1549 Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1550 reset_token = true;
1551 self.observer
1552 .event("token_reset", "change token expired; full replay");
1553 self.store.mutate(|l| l.change_token = None)?;
1554 continue 'restart;
1555 }
1556 Err(SyncError::ZoneMissing) if !recreated => {
1557 recreated = true;
1558 self.observer
1559 .event("zone_recreated", "zone missing; re-preparing");
1560 self.store.mutate(|l| {
1564 l.change_token = None;
1565 l.remote_versions.clear();
1566 l.aliases.clear();
1567 })?;
1568 self.backend.prepare()?;
1569 continue 'restart;
1570 }
1571 Err(e) => return Err(e),
1572 }
1573 }
1574 }
1575 }
1576
1577 fn flush_tombstones(
1580 &mut self,
1581 canonical: &BTreeMap<String, RemoteEntry>,
1582 generation: u64,
1583 rep: &mut PassReport,
1584 ) -> SyncResult<bool> {
1585 let ids: Vec<String> = self
1586 .store
1587 .ledger()
1588 .pending_tombstones
1589 .iter()
1590 .cloned()
1591 .collect();
1592 if ids.is_empty() {
1593 return Ok(true);
1594 }
1595 struct Write {
1596 id: String,
1597 record: RawRecord,
1598 }
1599 let mut writes: Vec<Write> = Vec::new();
1600 let mut never_remote: Vec<String> = Vec::new();
1601 for id in &ids {
1602 let mut names: BTreeSet<String> = self
1603 .store
1604 .ledger()
1605 .aliases
1606 .get(id)
1607 .cloned()
1608 .unwrap_or_default();
1609 let known =
1610 self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1611 if names.is_empty() && !known {
1612 never_remote.push(id.clone());
1613 continue;
1614 }
1615 names.insert(self.mapper.record_name(id));
1616 let at = self
1617 .store
1618 .ledger()
1619 .tombstones
1620 .get(id)
1621 .copied()
1622 .unwrap_or_else(|| self.clock.now_ms());
1623 for name in names {
1624 let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1625 Some(e) => e.record.clone(),
1626 None => self
1627 .backend
1628 .fetch_record(&name)
1629 .ok()
1630 .flatten()
1631 .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1632 };
1633 write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1634 writes.push(Write {
1635 id: id.clone(),
1636 record,
1637 });
1638 }
1639 }
1640 if !never_remote.is_empty() {
1641 self.store.mutate(|l| {
1643 for id in never_remote {
1644 l.pending_tombstones.remove(&id);
1645 }
1646 })?;
1647 }
1648 let mut failed: BTreeSet<String> = BTreeSet::new();
1649 let size = self
1650 .config
1651 .tombstone_batch
1652 .min(self.backend.max_batch())
1653 .max(1);
1654 for chunk in writes.chunks(size) {
1655 self.check_current(generation)?;
1656 let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1657 let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1658 for w in chunk {
1659 let ok = outcomes
1660 .iter()
1661 .find(|o| o.record_name == w.record.record_name)
1662 .is_some_and(|o| o.result.is_ok());
1663 if !ok {
1664 failed.insert(w.id.clone());
1665 self.observer
1666 .event("tombstone_save_failed", &w.record.record_name);
1667 }
1668 }
1669 }
1670 let acked: Vec<String> = writes
1671 .iter()
1672 .map(|w| w.id.clone())
1673 .collect::<BTreeSet<_>>()
1674 .into_iter()
1675 .filter(|id| !failed.contains(id))
1676 .collect();
1677 rep.tombstones_written += acked.len();
1678 self.store.mutate(|l| {
1679 for id in &acked {
1680 l.pending_tombstones.remove(id);
1681 }
1682 })?;
1683 Ok(failed.is_empty())
1684 }
1685
1686 fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1688 for chunk in records.chunks(self.backend.max_batch().max(1)) {
1689 for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1690 o.result?;
1691 }
1692 }
1693 Ok(())
1694 }
1695}