Skip to main content

rightkit_sync/
engine.rs

1//! The sync pass. One `run_pass` mirrors the iOS `sync()` actor method and the
2//! desktop `Coordinator::sync_pass`; see API.md for the shared contract.
3
4use std::collections::{BTreeMap, BTreeSet, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17    /// Upload chunk size for edits to records that already exist remotely.
18    pub batch_size: usize,
19    /// Tombstone write chunk size.
20    pub tombstone_batch: usize,
21    /// Publish never-seen items one record per save, oldest first, and stop
22    /// new publication for the pass after the first failure (preserves
23    /// history order: CloudKit gives no ordering inside one operation).
24    pub ordered_new_publication: bool,
25    /// Treat a known-published item that vanished from the local store as a
26    /// local deletion (desktop). Apps with explicit `note_deleted` calls
27    /// (iOS) leave this off.
28    pub infer_local_deletions: bool,
29    pub backoff: BackoffPolicy,
30}
31
32impl Default for SyncConfig {
33    fn default() -> Self {
34        Self {
35            batch_size: 200,
36            tombstone_batch: 50,
37            ordered_new_publication: true,
38            infer_local_deletions: false,
39            backoff: BackoffPolicy::default(),
40        }
41    }
42}
43
44#[derive(Debug, Clone, PartialEq)]
45pub enum PassOutcome {
46    /// Everything applied; token advanced; no pending work.
47    Synced,
48    /// Token advanced but some uploads/tombstones/applies remain queued.
49    Pending(String),
50    /// Backoff window still open; nothing was attempted.
51    Deferred {
52        secs: f64,
53    },
54    Unavailable(String),
55    Disabled,
56    /// Auth generation changed or sync was switched off mid-pass.
57    Cancelled,
58    Failed(String),
59}
60
61#[derive(Debug, Clone, PartialEq)]
62pub struct PassReport {
63    pub outcome: PassOutcome,
64    pub uploaded: usize,
65    pub downloaded: usize,
66    pub tombstones_applied: usize,
67    pub tombstones_written: usize,
68    pub quarantined: usize,
69    pub blocked_reuploads: usize,
70    /// Seconds until the next pass should run, when one is scheduled.
71    pub retry_after_secs: Option<f64>,
72}
73
74impl PassReport {
75    fn new() -> Self {
76        Self {
77            outcome: PassOutcome::Synced,
78            uploaded: 0,
79            downloaded: 0,
80            tombstones_applied: 0,
81            tombstones_written: 0,
82            quarantined: 0,
83            blocked_reuploads: 0,
84            retry_after_secs: None,
85        }
86    }
87}
88
89/// Cumulative engine counters and current ledger queue sizes, from [`SyncEngine::metrics`].
90/// Queue sizes are the figures that grow resident memory: the ledger keeps one remote
91/// version per record and is serialized in full on every durable write.
92#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
93pub struct SyncMetrics {
94    /// Passes run since the engine was built (including deferred ones).
95    pub passes: u64,
96    /// Passes that ended `Failed` or `Unavailable`.
97    pub failed_passes: u64,
98    /// Wall-clock duration of the most recent pass, in milliseconds.
99    pub last_pass_ms: u64,
100    pub downloaded_total: u64,
101    pub uploaded_total: u64,
102    pub tombstones_applied_total: u64,
103    pub tombstones_written_total: u64,
104    /// Remote versions cached in the ledger (one per known record).
105    pub remote_versions: u64,
106    /// Tombstones the ledger blocks re-import of, forever.
107    pub tombstones: u64,
108    pub pending_uploads: u64,
109    pub pending_tombstones: u64,
110    /// Downloaded records whose local apply was deferred.
111    pub pending_apply: u64,
112}
113
114/// Runtime figures of the engine, computed on call by [`SyncEngine::status`].
115#[derive(Debug, Clone, Copy, PartialEq)]
116pub struct SyncStatus {
117    /// Pass counters and the ledger queue sizes (`pending_*`, `remote_versions`).
118    pub metrics: SyncMetrics,
119    /// Resident memory, CPU time and I/O counters of the whole process (the engine runs in it and
120    /// has no thread of its own), or `None` when the host cannot report them. See
121    /// `rightkit_child_process::ProcessUsage`.
122    pub process: Option<rightkit_child_process::ProcessUsage>,
123}
124
125#[derive(Debug, Default)]
126struct Counters {
127    passes: u64,
128    failed_passes: u64,
129    last_pass_ms: u64,
130    downloaded: u64,
131    uploaded: u64,
132    tombstones_applied: u64,
133    tombstones_written: u64,
134}
135
136const MAX_CONFLICT_ROUNDS: usize = 4;
137
138enum Resolved {
139    /// Re-saved on top of the server's version.
140    Saved,
141    /// The server version stands (applied locally, tombstoned, or equal).
142    RemoteWon,
143    Failed(String),
144}
145
146enum RemoteState<T> {
147    Live(T),
148    Tombstone,
149}
150
151struct RemoteEntry<T> {
152    id: String,
153    record_name: String,
154    /// updatedAt for live records, deletedAt for tombstones.
155    version_ms: i64,
156    state: RemoteState<T>,
157    record: RawRecord,
158}
159
160impl<T> RemoteEntry<T> {
161    fn is_tombstone(&self) -> bool {
162        matches!(self.state, RemoteState::Tombstone)
163    }
164    /// Total order for alias reduction: tombstone > live, then newer
165    /// version, then larger record name (deterministic on every device).
166    fn rank(&self) -> (bool, i64, &str) {
167        (
168            self.is_tombstone(),
169            self.version_ms,
170            self.record_name.as_str(),
171        )
172    }
173}
174
175pub struct SyncEngine<M: RecordMapper> {
176    mapper: Arc<M>,
177    backend: Arc<dyn Backend>,
178    auth: Arc<dyn AuthProvider>,
179    local: Arc<dyn LocalStore<M::Item>>,
180    policy: Box<dyn ConflictPolicy<M::Item>>,
181    clock: Arc<dyn Clock>,
182    observer: Arc<dyn SyncObserver>,
183    hooks: Arc<dyn PassHooks>,
184    config: SyncConfig,
185    store: LedgerStore,
186    replay_flags: Vec<String>,
187    counters: Counters,
188}
189
190pub struct SyncEngineBuilder<M: RecordMapper> {
191    mapper: Arc<M>,
192    backend: Arc<dyn Backend>,
193    local: Arc<dyn LocalStore<M::Item>>,
194    storage: Box<dyn LedgerStorage>,
195    auth: Arc<dyn AuthProvider>,
196    policy: Box<dyn ConflictPolicy<M::Item>>,
197    clock: Arc<dyn Clock>,
198    observer: Arc<dyn SyncObserver>,
199    hooks: Arc<dyn PassHooks>,
200    config: SyncConfig,
201}
202
203impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
204    pub fn new(
205        mapper: Arc<M>,
206        backend: Arc<dyn Backend>,
207        local: Arc<dyn LocalStore<M::Item>>,
208        storage: Box<dyn LedgerStorage>,
209    ) -> Self
210    where
211        M::Item: 'static,
212    {
213        Self {
214            mapper,
215            backend,
216            local,
217            storage,
218            auth: Arc::new(AlwaysAvailable),
219            policy: Box::new(LastWriterWins),
220            clock: Arc::new(SystemClock),
221            observer: Arc::new(NullObserver),
222            hooks: Arc::new(NoHooks),
223            config: SyncConfig::default(),
224        }
225    }
226    pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
227        self.auth = v;
228        self
229    }
230    pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
231        self.policy = v;
232        self
233    }
234    pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
235        self.clock = v;
236        self
237    }
238    pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
239        self.observer = v;
240        self
241    }
242    pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
243        self.hooks = v;
244        self
245    }
246    pub fn config(mut self, v: SyncConfig) -> Self {
247        self.config = v;
248        self
249    }
250    pub fn build(self) -> SyncResult<SyncEngine<M>> {
251        let store = LedgerStore::open(self.storage)?;
252        if store.recovered_from_corruption {
253            self.observer
254                .event("ledger_recovered", "ledger unreadable; full replay");
255        }
256        Ok(SyncEngine {
257            mapper: self.mapper,
258            backend: self.backend,
259            auth: self.auth,
260            local: self.local,
261            policy: self.policy,
262            clock: self.clock,
263            observer: self.observer,
264            hooks: self.hooks,
265            config: self.config,
266            store,
267            replay_flags: Vec::new(),
268            counters: Counters::default(),
269        })
270    }
271}
272
273struct Fetched {
274    records: Vec<RawRecord>,
275    deleted: Vec<String>,
276    token: Option<Vec<u8>>,
277}
278
279impl<M: RecordMapper + 'static> SyncEngine<M>
280where
281    M::Item: 'static,
282{
283    pub fn ledger(&self) -> &crate::ledger::Ledger {
284        self.store.ledger()
285    }
286
287    /// Offline queue: record that `id` changed locally. Durable immediately.
288    pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
289        let id = id.to_string();
290        self.store.mutate(|l| {
291            if !l.tombstones.contains_key(&id) {
292                l.pending_uploads.insert(id);
293            }
294        })
295    }
296
297    /// Local deletion: block the id and queue a tombstone write. Durable.
298    pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
299        let now = self.clock.now_ms();
300        let id = id.to_string();
301        self.store.mutate(|l| {
302            l.bury(&id, now);
303            l.pending_tombstones.insert(id);
304        })
305    }
306
307    /// Replay the whole zone once for `flag` (one-shot repair migrations).
308    pub fn request_full_replay_once(&mut self, flag: &str) {
309        if !self.store.ledger().flags.contains(flag) {
310            self.replay_flags.push(flag.to_string());
311        }
312    }
313
314    /// Runs one sync pass and records it in [`SyncEngine::metrics`].
315    pub fn run_pass(&mut self) -> PassReport {
316        let started = Instant::now();
317        let rep = self.run_pass_once();
318        self.record_pass(&rep, started.elapsed());
319        rep
320    }
321
322    fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
323        let c = &mut self.counters;
324        c.passes += 1;
325        if matches!(
326            rep.outcome,
327            PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
328        ) {
329            c.failed_passes += 1;
330        }
331        c.last_pass_ms = elapsed.as_millis() as u64;
332        c.downloaded += rep.downloaded as u64;
333        c.uploaded += rep.uploaded as u64;
334        c.tombstones_applied += rep.tombstones_applied as u64;
335        c.tombstones_written += rep.tombstones_written as u64;
336    }
337
338    /// Pass counters, ledger queue sizes and process resource usage, read now. No polling thread
339    /// runs for this.
340    pub fn status(&self) -> SyncStatus {
341        SyncStatus {
342            metrics: self.metrics(),
343            process: rightkit_child_process::process_usage().ok(),
344        }
345    }
346
347    /// Cumulative pass counters and current ledger queue sizes.
348    pub fn metrics(&self) -> SyncMetrics {
349        let l = self.store.ledger();
350        let c = &self.counters;
351        SyncMetrics {
352            passes: c.passes,
353            failed_passes: c.failed_passes,
354            last_pass_ms: c.last_pass_ms,
355            downloaded_total: c.downloaded,
356            uploaded_total: c.uploaded,
357            tombstones_applied_total: c.tombstones_applied,
358            tombstones_written_total: c.tombstones_written,
359            remote_versions: l.remote_versions.len() as u64,
360            tombstones: l.tombstones.len() as u64,
361            pending_uploads: l.pending_uploads.len() as u64,
362            pending_tombstones: l.pending_tombstones.len() as u64,
363            pending_apply: l.pending_apply.len() as u64,
364        }
365    }
366
367    fn run_pass_once(&mut self) -> PassReport {
368        let mut rep = PassReport::new();
369        if !self.auth.enabled() {
370            rep.outcome = PassOutcome::Disabled;
371            return rep;
372        }
373        let now = self.clock.now_ms();
374        if let Some(until) = self.store.ledger().retry_after_ms {
375            if until > now {
376                let secs = (until - now) as f64 / 1000.0;
377                rep.outcome = PassOutcome::Deferred { secs };
378                rep.retry_after_secs = Some(secs);
379                return rep;
380            }
381        }
382        let generation = self.auth.generation();
383        match self.pass_inner(generation, &mut rep) {
384            Ok(pending) => {
385                let result = self.store.mutate(|l| {
386                    if pending.is_none() {
387                        l.consecutive_failures = 0;
388                        l.retry_after_ms = None;
389                    }
390                });
391                match (result, pending) {
392                    (Err(e), _) => self.fail(e, &mut rep),
393                    (Ok(()), None) => rep.outcome = PassOutcome::Synced,
394                    (Ok(()), Some(why)) => {
395                        let secs = self.config.backoff.pending_floor_secs;
396                        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
397                        let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
398                        rep.retry_after_secs = Some(secs);
399                        rep.outcome = PassOutcome::Pending(why);
400                    }
401                }
402            }
403            Err(e) => self.fail(e, &mut rep),
404        }
405        rep
406    }
407
408    fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
409        if e == SyncError::Cancelled {
410            rep.outcome = PassOutcome::Cancelled;
411            return;
412        }
413        self.observer.event("pass_failed", &e.to_string());
414        let failures = self.store.ledger().consecutive_failures.saturating_add(1);
415        let secs = self
416            .config
417            .backoff
418            .delay_secs(failures, e.retry_hint_secs());
419        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
420        let _ = self.store.mutate(|l| {
421            l.consecutive_failures = failures;
422            l.retry_after_ms = Some(until);
423        });
424        rep.retry_after_secs = Some(secs);
425        rep.outcome = match e {
426            SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
427            other => PassOutcome::Failed(other.to_string()),
428        };
429    }
430
431    /// A conditional save lost a race. Read the server record, decide with
432    /// the conflict policy (tombstone always wins) and either re-save on top
433    /// of the server's change tag, take the remote, or keep both unchanged.
434    /// Bounded: a record that keeps changing under us stays pending.
435    fn resolve_conflict(
436        &mut self,
437        header: &ItemHeader,
438        item: &M::Item,
439        name: &str,
440        mut server: Option<RawRecord>,
441        generation: u64,
442        rep: &mut PassReport,
443    ) -> SyncResult<Resolved> {
444        for _ in 0..MAX_CONFLICT_ROUNDS {
445            self.check_current(generation)?;
446            let server_rec = match server.take() {
447                Some(r) => r,
448                None => match self.backend.fetch_record(name)? {
449                    Some(r) => r,
450                    None => return Ok(Resolved::Failed("conflicting record vanished".into())),
451                },
452            };
453            self.observer.event("save_conflict", name);
454            if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
455                self.local.remove(&id)?;
456                self.store.mutate(|l| l.bury(&id, deleted_at))?;
457                rep.tombstones_applied += 1;
458                return Ok(Resolved::RemoteWon);
459            }
460            let remote_item = match self.mapper.decode(&server_rec) {
461                Ok(i) => i,
462                Err(e) => {
463                    rep.quarantined += 1;
464                    return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
465                }
466            };
467            let rh = self.mapper.header(&remote_item);
468            if self
469                .policy
470                .upload_local_over_remote(header, item, &rh, &remote_item)
471            {
472                let mut rec = server_rec;
473                rec.set(field::ID, FieldValue::Text(header.id.clone()));
474                rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
475                rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
476                rec.fields.remove(field::DELETED_AT);
477                self.mapper.encode(item, &mut rec);
478                let outcomes = self
479                    .backend
480                    .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
481                let Some(o) = outcomes.into_iter().next() else {
482                    return Ok(Resolved::Failed("no save outcome".into()));
483                };
484                match o.result {
485                    Ok(()) => return Ok(Resolved::Saved),
486                    Err(SyncError::Conflict) => server = o.server_record,
487                    Err(e) => return Ok(Resolved::Failed(e.to_string())),
488                }
489                continue;
490            }
491            let version = RemoteVersion {
492                record_name: name.to_string(),
493                updated_at_ms: rh.updated_at_ms,
494            };
495            if self.policy.accept_remote_over_local(header, &rh) {
496                self.check_current(generation)?;
497                match self.local.apply_remote(&[remote_item])?.into_iter().next() {
498                    Some(ApplyOutcome::Deferred(_)) => {
499                        let id = header.id.clone();
500                        self.store.mutate(|l| {
501                            l.pending_apply.insert(id, server_rec);
502                        })?;
503                    }
504                    _ => rep.downloaded += 1,
505                }
506            }
507            let id = header.id.clone();
508            self.store.mutate(|l| {
509                l.pending_uploads.remove(&id);
510                l.remote_versions.insert(id, version);
511            })?;
512            return Ok(Resolved::RemoteWon);
513        }
514        Ok(Resolved::Failed("conflict retries exhausted".into()))
515    }
516
517    /// Bind the ledger to the signed-in account scope; reset it when the
518    /// scope changed so one account's cursor, versions and queued deletions
519    /// are never applied to another.
520    fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
521        self.check_current(generation)?;
522        let Some(scope) = self.auth.account_scope() else {
523            return Ok(());
524        };
525        match self.store.ledger().scope.clone() {
526            Some(bound) if bound == scope => Ok(()),
527            None => {
528                // Ledger from before scoping: keep tombstones (they only
529                // suppress local copies); never import its cursor, versions,
530                // aliases or queued deletions into this account.
531                let dropped = {
532                    let l = self.store.ledger();
533                    l.change_token.is_some()
534                        || !l.remote_versions.is_empty()
535                        || !l.aliases.is_empty()
536                        || !l.pending_tombstones.is_empty()
537                };
538                self.store.mutate(|l| {
539                    l.scope = Some(scope);
540                    l.change_token = None;
541                    l.remote_versions.clear();
542                    l.aliases.clear();
543                    l.pending_tombstones.clear();
544                    l.has_synced = false;
545                })?;
546                if dropped {
547                    self.observer
548                        .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
549                }
550                Ok(())
551            }
552            Some(bound) => {
553                self.hooks.on_account_switch(&bound, &scope);
554                self.store.mutate(|l| {
555                    *l = crate::ledger::Ledger::default();
556                    l.scope = Some(scope.clone());
557                })?;
558                self.observer.event("account_switched", "ledger reset");
559                Ok(())
560            }
561        }
562    }
563
564    fn check_current(&self, generation: u64) -> SyncResult<()> {
565        if self.auth.enabled() && self.auth.generation() == generation {
566            Ok(())
567        } else {
568            Err(SyncError::Cancelled)
569        }
570    }
571
572    /// Returns `Some(reason)` when the pass finished with queued work left.
573    fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
574        self.auth.availability()?;
575        // `availability` refreshes the account identity, which may bump the
576        // generation; the pass belongs to the generation observed after it.
577        let generation = self.auth.generation().max(generation);
578        self.bind_account(generation)?;
579        self.backend.prepare()?;
580        self.check_current(generation)?;
581
582        // Domain announce-style records go first; failure fails the pass.
583        let mut extras = self.store.ledger().extras.clone();
584        let outbound = self.hooks.outbound(&mut extras);
585        if !outbound.is_empty() {
586            self.check_current(generation)?;
587            self.write_required(&outbound)?;
588        }
589        self.check_current(generation)?;
590        self.store.mutate(|l| l.extras = extras)?;
591
592        // One-shot full replays (repair migrations) reset only the cursor.
593        if !self.replay_flags.is_empty() {
594            self.store.mutate(|l| l.change_token = None)?;
595        }
596
597        let fetched = self.fetch_all(generation)?;
598        let mapper_type = self.mapper.record_type().to_string();
599        let mut entries: Vec<RemoteEntry<M::Item>> = Vec::new();
600        let mut foreign: Vec<RawRecord> = Vec::new();
601        for record in fetched.records {
602            if record.record_type != mapper_type {
603                foreign.push(record);
604                continue;
605            }
606            if let Some((id, deleted_at)) = read_tombstone(&record) {
607                entries.push(RemoteEntry {
608                    id,
609                    record_name: record.record_name.clone(),
610                    version_ms: deleted_at,
611                    state: RemoteState::Tombstone,
612                    record,
613                });
614                continue;
615            }
616            match self.mapper.decode(&record) {
617                Ok(item) => {
618                    let h = self.mapper.header(&item);
619                    entries.push(RemoteEntry {
620                        id: h.id,
621                        record_name: record.record_name.clone(),
622                        version_ms: h.updated_at_ms,
623                        state: RemoteState::Live(item),
624                        record,
625                    });
626                }
627                Err(why) => {
628                    // A bad record never blocks the page (both platforms).
629                    rep.quarantined += 1;
630                    self.observer.event(
631                        "record_quarantined",
632                        &format!("{}: {why}", record.record_name),
633                    );
634                }
635            }
636        }
637        // Server-side hard deletions of records we know behave as tombstones.
638        for name in &fetched.deleted {
639            let known = self
640                .store
641                .ledger()
642                .aliases
643                .iter()
644                .find(|(_, names)| names.contains(name))
645                .map(|(id, _)| id.clone())
646                .or_else(|| {
647                    self.store
648                        .ledger()
649                        .remote_versions
650                        .contains_key(name)
651                        .then(|| name.clone())
652                });
653            if let Some(id) = known {
654                if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
655                    let mut record = RawRecord::new(&mapper_type, name);
656                    let now = self.clock.now_ms();
657                    write_tombstone(&mut record, &id, now, &[]);
658                    entries.push(RemoteEntry {
659                        id,
660                        record_name: name.clone(),
661                        version_ms: now,
662                        state: RemoteState::Tombstone,
663                        record,
664                    });
665                }
666            }
667        }
668        if rep.quarantined > 0 {
669            let n = rep.quarantined as u64;
670            self.store.mutate(|l| l.quarantined_total += n)?;
671        }
672
673        // Aliases: every record name ever seen for a logical id.
674        {
675            let seen: Vec<(String, String)> = entries
676                .iter()
677                .map(|e| (e.id.clone(), e.record_name.clone()))
678                .collect();
679            self.store.mutate(|l| {
680                for (id, name) in seen {
681                    l.aliases.entry(id).or_default().insert(name);
682                }
683            })?;
684        }
685
686        // Reduce duplicate records of one logical id.
687        let mut canonical: BTreeMap<String, RemoteEntry<M::Item>> = BTreeMap::new();
688        for entry in entries {
689            match canonical.get(&entry.id) {
690                Some(cur) if cur.rank() >= entry.rank() => {}
691                _ => {
692                    canonical.insert(entry.id.clone(), entry);
693                }
694            }
695        }
696
697        // Domain-owned record types (queue requests etc.).
698        if !foreign.is_empty() {
699            let mut extras = self.store.ledger().extras.clone();
700            let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
701            // Responses are durable on the zone BEFORE the extras that record
702            // them are persisted, and a failed write aborts the pass before
703            // the change token can pass the request: the delta replays and
704            // the idempotent hook answers again. Never acknowledge a request
705            // whose response never landed.
706            if !responses.is_empty() {
707                self.check_current(generation)?;
708                if let Err(e) = self.write_required(&responses) {
709                    self.observer.event("foreign_write_failed", &e.to_string());
710                    return Err(e);
711                }
712            }
713            self.check_current(generation)?;
714            self.store.mutate(|l| l.extras = extras)?;
715        }
716
717        // Remote tombstones are authoritative: delete locally, block id.
718        let tombs: Vec<(String, i64)> = canonical
719            .values()
720            .filter(|e| e.is_tombstone())
721            .map(|e| (e.id.clone(), e.version_ms))
722            .collect();
723        for (id, at) in &tombs {
724            self.check_current(generation)?;
725            self.local.remove(id)?;
726            let (id, at) = (id.clone(), *at);
727            self.store.mutate(|l| {
728                l.bury(&id, at);
729                l.pending_tombstones.remove(&id);
730            })?;
731        }
732        rep.tombstones_applied = tombs.len();
733        self.check_current(generation)?;
734
735        let mut pending: Vec<String> = Vec::new();
736
737        // Our own queued deletions.
738        match self.flush_tombstones(&canonical, generation, rep) {
739            Ok(true) => {}
740            Ok(false) => pending.push("tombstones pending".into()),
741            Err(e) => return Err(e),
742        }
743
744        // Inferred deletions (desktop): published item vanished locally.
745        let local_items = self.local.all()?;
746        let mut local_ids: HashSet<String> = HashSet::new();
747        let mut locals: Vec<(ItemHeader, M::Item)> = Vec::new();
748        for item in local_items {
749            let h = self.mapper.header(&item);
750            local_ids.insert(h.id.clone());
751            locals.push((h, item));
752        }
753        if self.config.infer_local_deletions {
754            let gone: Vec<String> = self
755                .store
756                .ledger()
757                .remote_versions
758                .keys()
759                .filter(|id| !local_ids.contains(*id))
760                .filter(|id| !self.store.ledger().tombstones.contains_key(*id))
761                .filter(|id| !self.store.ledger().pending_apply.contains_key(*id))
762                .cloned()
763                .collect();
764            if !gone.is_empty() {
765                let now = self.clock.now_ms();
766                self.store.mutate(|l| {
767                    for id in gone {
768                        l.bury(&id, now);
769                        l.pending_tombstones.insert(id);
770                    }
771                })?;
772                if !self.flush_tombstones(&canonical, generation, rep)? {
773                    pending.push("tombstones pending".into());
774                }
775            }
776        }
777
778        // ---- Uploads ----
779        let blocked: BTreeSet<String> = self.store.ledger().tombstones.keys().cloned().collect();
780        // Drop queued ids whose item no longer exists locally.
781        let stale: Vec<String> = self
782            .store
783            .ledger()
784            .pending_uploads
785            .iter()
786            .filter(|id| !local_ids.contains(*id))
787            .cloned()
788            .collect();
789        if !stale.is_empty() {
790            self.store.mutate(|l| {
791                for id in stale {
792                    l.pending_uploads.remove(&id);
793                }
794            })?;
795        }
796
797        struct Upload<I> {
798            header: ItemHeader,
799            item: I,
800            base: Option<RawRecord>,
801            is_new: bool,
802        }
803        let mut uploads: Vec<Upload<M::Item>> = Vec::new();
804        let mut failed_uploads = 0usize;
805        for (header, item) in &locals {
806            if blocked.contains(&header.id) {
807                rep.blocked_reuploads += 1;
808                continue;
809            }
810            let queued = self.store.ledger().pending_uploads.contains(&header.id);
811            match canonical.get(&header.id) {
812                Some(RemoteEntry {
813                    state: RemoteState::Live(remote_item),
814                    record,
815                    ..
816                }) => {
817                    let rh = self.mapper.header(remote_item);
818                    if self
819                        .policy
820                        .upload_local_over_remote(header, item, &rh, remote_item)
821                    {
822                        uploads.push(Upload {
823                            header: header.clone(),
824                            item: item.clone(),
825                            base: Some(record.clone()),
826                            is_new: false,
827                        });
828                    }
829                }
830                Some(_) => {}
831                None => {
832                    if let Some(known) =
833                        self.store.ledger().remote_versions.get(&header.id).cloned()
834                    {
835                        // Cursor delta did not include it: unchanged remotely.
836                        if known.updated_at_ms < header.updated_at_ms || queued {
837                            match self.backend.fetch_record(&known.record_name) {
838                                Ok(base) => uploads.push(Upload {
839                                    header: header.clone(),
840                                    item: item.clone(),
841                                    base,
842                                    is_new: false,
843                                }),
844                                Err(e) => {
845                                    failed_uploads += 1;
846                                    self.observer.event("record_fetch_failed", &e.to_string());
847                                }
848                            }
849                        }
850                    } else {
851                        uploads.push(Upload {
852                            header: header.clone(),
853                            item: item.clone(),
854                            base: None,
855                            is_new: true,
856                        });
857                    }
858                }
859            }
860        }
861        // Oldest origin first, ties by record name: deterministic history.
862        uploads.sort_by(|a, b| {
863            (
864                a.header.created_at_ms,
865                self.mapper.record_name(&a.header.id),
866            )
867                .cmp(&(
868                    b.header.created_at_ms,
869                    self.mapper.record_name(&b.header.id),
870                ))
871        });
872        let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
873        let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
874        let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
875        if self.config.ordered_new_publication {
876            chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
877            let mut it = existing.into_iter().peekable();
878            while it.peek().is_some() {
879                chunks.push((false, it.by_ref().take(size).collect()));
880            }
881        } else {
882            let mut it = new_ones.into_iter().chain(existing).peekable();
883            while it.peek().is_some() {
884                chunks.push((false, it.by_ref().take(size).collect()));
885            }
886        }
887
888        let mut blocked_new = false;
889        for (ordered_new, chunk) in chunks {
890            if ordered_new && blocked_new {
891                continue;
892            }
893            self.check_current(generation)?;
894            let mut records = Vec::with_capacity(chunk.len());
895            let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
896            for u in &chunk {
897                let name = self.mapper.record_name(&u.header.id);
898                let mut rec = u
899                    .base
900                    .clone()
901                    .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
902                rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
903                rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
904                rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
905                rec.fields.remove(field::DELETED_AT);
906                self.mapper.encode(&u.item, &mut rec);
907                by_name.insert(
908                    rec.record_name.clone(),
909                    (u.header.id.clone(), u.header.updated_at_ms),
910                );
911                records.push(rec);
912            }
913            // Conditional: an edit based on a stale change tag (or an insert of
914            // a name that now exists) is refused, never overwritten.
915            let outcomes = self
916                .backend
917                .save_records(&records, SavePolicy::IfServerUnchanged)?;
918            let mut saved: Vec<(String, String, i64)> = Vec::new();
919            let mut chunk_failed = false;
920            for o in outcomes {
921                let Some((id, version)) = by_name.get(&o.record_name) else {
922                    continue;
923                };
924                match o.result {
925                    Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
926                    Err(SyncError::Conflict) => {
927                        let upload = chunk
928                            .iter()
929                            .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
930                            .expect("outcome names an upload of this chunk");
931                        let (header, item) = (upload.header.clone(), upload.item.clone());
932                        match self.resolve_conflict(
933                            &header,
934                            &item,
935                            &o.record_name,
936                            o.server_record,
937                            generation,
938                            rep,
939                        )? {
940                            Resolved::Saved => {
941                                saved.push((id.clone(), o.record_name.clone(), *version))
942                            }
943                            Resolved::RemoteWon => {}
944                            Resolved::Failed(why) => {
945                                chunk_failed = true;
946                                failed_uploads += 1;
947                                self.observer.event(
948                                    "record_save_failed",
949                                    &format!("{}: {why}", o.record_name),
950                                );
951                            }
952                        }
953                    }
954                    Err(e) => {
955                        chunk_failed = true;
956                        failed_uploads += 1;
957                        self.observer
958                            .event("record_save_failed", &format!("{}: {e}", o.record_name));
959                    }
960                }
961            }
962            if ordered_new && chunk_failed {
963                blocked_new = true;
964            }
965            // Persist progress per chunk: a crash resumes from here.
966            rep.uploaded += saved.len();
967            self.store.mutate(|l| {
968                for (id, name, version) in saved {
969                    l.pending_uploads.remove(&id);
970                    l.aliases
971                        .entry(id.clone())
972                        .or_default()
973                        .insert(name.clone());
974                    l.remote_versions.insert(
975                        id,
976                        RemoteVersion {
977                            record_name: name,
978                            updated_at_ms: version,
979                        },
980                    );
981                }
982            })?;
983        }
984        if failed_uploads > 0 {
985            pending.push(format!("{failed_uploads} upload(s) pending"));
986        }
987
988        // ---- Downloads ----
989        let mut downloads: Vec<(ItemHeader, M::Item, RawRecord)> = Vec::new();
990        let mut seen_ids: HashSet<String> = HashSet::new();
991        for entry in canonical.values() {
992            let RemoteState::Live(item) = &entry.state else {
993                continue;
994            };
995            if self.store.ledger().tombstones.contains_key(&entry.id) {
996                continue;
997            }
998            let rh = self.mapper.header(item);
999            let take = match locals.iter().find(|(h, _)| h.id == entry.id) {
1000                None => true,
1001                Some((lh, _)) => self.policy.accept_remote_over_local(lh, &rh),
1002            };
1003            if take {
1004                seen_ids.insert(entry.id.clone());
1005                downloads.push((rh, item.clone(), entry.record.clone()));
1006            }
1007        }
1008        // Retry records whose local apply was deferred earlier.
1009        let stashed: Vec<(String, RawRecord)> = self
1010            .store
1011            .ledger()
1012            .pending_apply
1013            .iter()
1014            .map(|(k, v)| (k.clone(), v.clone()))
1015            .collect();
1016        for (id, record) in stashed {
1017            if seen_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1018                continue;
1019            }
1020            if let Ok(item) = self.mapper.decode(&record) {
1021                downloads.push((self.mapper.header(&item), item, record));
1022            }
1023        }
1024        downloads.sort_by(|a, b| {
1025            (a.0.created_at_ms, a.0.id.as_str()).cmp(&(b.0.created_at_ms, b.0.id.as_str()))
1026        });
1027        if !downloads.is_empty() {
1028            self.check_current(generation)?;
1029            let items: Vec<M::Item> = downloads.iter().map(|d| d.1.clone()).collect();
1030            let outcomes = self.local.apply_remote(&items)?;
1031            if outcomes.len() != downloads.len() {
1032                return Err(SyncError::Persistence(
1033                    "apply_remote returned wrong outcome count".into(),
1034                ));
1035            }
1036            let mut deferred = 0usize;
1037            let mut applied_ids: Vec<String> = Vec::new();
1038            let mut stash: Vec<(String, RawRecord)> = Vec::new();
1039            for ((h, _, record), out) in downloads.into_iter().zip(outcomes) {
1040                match out {
1041                    ApplyOutcome::Applied => {
1042                        rep.downloaded += 1;
1043                        applied_ids.push(h.id);
1044                    }
1045                    ApplyOutcome::Deferred(why) => {
1046                        deferred += 1;
1047                        self.observer
1048                            .event("apply_deferred", &format!("{}: {why}", h.id));
1049                        stash.push((h.id, record));
1050                    }
1051                }
1052            }
1053            self.store.mutate(|l| {
1054                for id in applied_ids {
1055                    l.pending_apply.remove(&id);
1056                }
1057                for (id, rec) in stash {
1058                    l.pending_apply.insert(id, rec);
1059                }
1060            })?;
1061            if deferred > 0 {
1062                pending.push(format!("{deferred} download(s) deferred"));
1063            }
1064        }
1065
1066        // Remote versions + advance the token: only now, after everything
1067        // above applied (or was durably queued).
1068        let versions: Vec<(String, RemoteVersion)> = canonical
1069            .values()
1070            .map(|e| {
1071                (
1072                    e.id.clone(),
1073                    RemoteVersion {
1074                        record_name: e.record_name.clone(),
1075                        updated_at_ms: e.version_ms,
1076                    },
1077                )
1078            })
1079            .collect();
1080        // Final guard: a pass whose account/enable generation changed during
1081        // I/O must not advance the cursor or mark the ledger synced.
1082        self.check_current(generation)?;
1083        let flags = std::mem::take(&mut self.replay_flags);
1084        let token = fetched.token;
1085        self.store.mutate(|l| {
1086            for (id, v) in versions {
1087                // A local upload this pass may already hold a newer version.
1088                let keep = l
1089                    .remote_versions
1090                    .get(&id)
1091                    .is_some_and(|k| k.updated_at_ms > v.updated_at_ms);
1092                if !keep {
1093                    l.remote_versions.insert(id, v);
1094                }
1095            }
1096            if token.is_some() {
1097                l.change_token = token;
1098            }
1099            l.has_synced = true;
1100            for f in flags {
1101                l.flags.insert(f);
1102            }
1103        })?;
1104        if rep.blocked_reuploads > 0 {
1105            self.observer
1106                .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1107        }
1108        Ok(if pending.is_empty() {
1109            None
1110        } else {
1111            Some(pending.join("; "))
1112        })
1113    }
1114
1115    /// Fetch every page; reset the token once on expiry and re-prepare the
1116    /// zone once if it vanished.
1117    fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1118        let mut reset_token = false;
1119        let mut recreated = false;
1120        'restart: loop {
1121            let mut token = self.store.ledger().change_token.clone();
1122            let mut out = Fetched {
1123                records: Vec::new(),
1124                deleted: Vec::new(),
1125                token: None,
1126            };
1127            loop {
1128                self.check_current(generation)?;
1129                match self.backend.fetch_changes(token.as_deref()) {
1130                    Ok(page) => {
1131                        out.records.extend(page.records);
1132                        out.deleted.extend(page.deleted_record_names);
1133                        if page.new_token.is_some() {
1134                            token = page.new_token.clone();
1135                            out.token = page.new_token;
1136                        }
1137                        if !page.more_coming {
1138                            return Ok(out);
1139                        }
1140                    }
1141                    Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1142                        reset_token = true;
1143                        self.observer
1144                            .event("token_reset", "change token expired; full replay");
1145                        self.store.mutate(|l| l.change_token = None)?;
1146                        continue 'restart;
1147                    }
1148                    Err(SyncError::ZoneMissing) if !recreated => {
1149                        recreated = true;
1150                        self.observer
1151                            .event("zone_recreated", "zone missing; re-preparing");
1152                        // Every remote record is gone: what the ledger believes is
1153                        // published no longer exists, so local items must
1154                        // re-publish (donor: acknowledgements are invalidated).
1155                        self.store.mutate(|l| {
1156                            l.change_token = None;
1157                            l.remote_versions.clear();
1158                            l.aliases.clear();
1159                        })?;
1160                        self.backend.prepare()?;
1161                        continue 'restart;
1162                    }
1163                    Err(e) => return Err(e),
1164                }
1165            }
1166        }
1167    }
1168
1169    /// Write queued tombstones to every alias record. Returns true when the
1170    /// queue drained.
1171    fn flush_tombstones(
1172        &mut self,
1173        canonical: &BTreeMap<String, RemoteEntry<M::Item>>,
1174        generation: u64,
1175        rep: &mut PassReport,
1176    ) -> SyncResult<bool> {
1177        let ids: Vec<String> = self
1178            .store
1179            .ledger()
1180            .pending_tombstones
1181            .iter()
1182            .cloned()
1183            .collect();
1184        if ids.is_empty() {
1185            return Ok(true);
1186        }
1187        struct Write {
1188            id: String,
1189            record: RawRecord,
1190        }
1191        let mut writes: Vec<Write> = Vec::new();
1192        let mut never_remote: Vec<String> = Vec::new();
1193        for id in &ids {
1194            let mut names: BTreeSet<String> = self
1195                .store
1196                .ledger()
1197                .aliases
1198                .get(id)
1199                .cloned()
1200                .unwrap_or_default();
1201            let known =
1202                self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1203            if names.is_empty() && !known {
1204                never_remote.push(id.clone());
1205                continue;
1206            }
1207            names.insert(self.mapper.record_name(id));
1208            let at = self
1209                .store
1210                .ledger()
1211                .tombstones
1212                .get(id)
1213                .copied()
1214                .unwrap_or_else(|| self.clock.now_ms());
1215            for name in names {
1216                let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1217                    Some(e) => e.record.clone(),
1218                    None => self
1219                        .backend
1220                        .fetch_record(&name)
1221                        .ok()
1222                        .flatten()
1223                        .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1224                };
1225                write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1226                writes.push(Write {
1227                    id: id.clone(),
1228                    record,
1229                });
1230            }
1231        }
1232        if !never_remote.is_empty() {
1233            // Created and deleted before ever reaching the zone: nothing to write.
1234            self.store.mutate(|l| {
1235                for id in never_remote {
1236                    l.pending_tombstones.remove(&id);
1237                }
1238            })?;
1239        }
1240        let mut failed: BTreeSet<String> = BTreeSet::new();
1241        let size = self
1242            .config
1243            .tombstone_batch
1244            .min(self.backend.max_batch())
1245            .max(1);
1246        for chunk in writes.chunks(size) {
1247            self.check_current(generation)?;
1248            let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1249            let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1250            for w in chunk {
1251                let ok = outcomes
1252                    .iter()
1253                    .find(|o| o.record_name == w.record.record_name)
1254                    .is_some_and(|o| o.result.is_ok());
1255                if !ok {
1256                    failed.insert(w.id.clone());
1257                    self.observer
1258                        .event("tombstone_save_failed", &w.record.record_name);
1259                }
1260            }
1261        }
1262        let acked: Vec<String> = writes
1263            .iter()
1264            .map(|w| w.id.clone())
1265            .collect::<BTreeSet<_>>()
1266            .into_iter()
1267            .filter(|id| !failed.contains(id))
1268            .collect();
1269        rep.tombstones_written += acked.len();
1270        self.store.mutate(|l| {
1271            for id in &acked {
1272                l.pending_tombstones.remove(id);
1273            }
1274        })?;
1275        Ok(failed.is_empty())
1276    }
1277
1278    /// Save records whose failure must fail the pass.
1279    fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1280        for chunk in records.chunks(self.backend.max_batch().max(1)) {
1281            for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1282                o.result?;
1283            }
1284        }
1285        Ok(())
1286    }
1287}