Skip to main content

rightkit_sync/
engine.rs

1//! The sync pass. One `run_pass` mirrors the iOS `sync()` actor method and the
2//! desktop `Coordinator::sync_pass`; see API.md for the shared contract.
3
4use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{ConsentRequest, LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17    /// Upload chunk size for edits to records that already exist remotely.
18    pub batch_size: usize,
19    /// Tombstone write chunk size.
20    pub tombstone_batch: usize,
21    /// Publish never-seen items one record per save, oldest first, and stop
22    /// new publication for the pass after the first failure (preserves
23    /// history order: CloudKit gives no ordering inside one operation).
24    pub ordered_new_publication: bool,
25    /// Treat a known-published item that vanished from the local store as a
26    /// local deletion (desktop). Apps with explicit `note_deleted` calls
27    /// (iOS) leave this off.
28    pub infer_local_deletions: bool,
29    /// When the signed-in account or scope differs from the one the ledger is bound to, pause
30    /// instead of resetting the ledger and publishing local history into the new account. The
31    /// pass reports [`PassOutcome::ConsentRequired`] and uploads, downloads and writes nothing
32    /// until the app calls [`SyncEngine::grant_consent`]. Default `false` (the earlier silent
33    /// reset), so existing apps keep their behaviour.
34    pub require_consent_on_scope_change: bool,
35    pub backoff: BackoffPolicy,
36}
37
38/// The app's answer to a [`PassOutcome::ConsentRequired`] pause.
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub enum ConsentChoice {
41    /// Publish this device's local items into the new account (and download its content).
42    Merge,
43    /// Keep this device's existing local items local: they are never published to the new
44    /// account. The new account's content is still downloaded; items created or received after
45    /// consent sync normally. Nothing local is deleted.
46    StartFresh,
47}
48
49impl Default for SyncConfig {
50    fn default() -> Self {
51        Self {
52            batch_size: 200,
53            tombstone_batch: 50,
54            ordered_new_publication: true,
55            infer_local_deletions: false,
56            require_consent_on_scope_change: false,
57            backoff: BackoffPolicy::default(),
58        }
59    }
60}
61
62#[derive(Debug, Clone, PartialEq)]
63pub enum PassOutcome {
64    /// Everything applied; token advanced; no pending work.
65    Synced,
66    /// Token advanced but some uploads/tombstones/applies remain queued.
67    Pending(String),
68    /// Backoff window still open; nothing was attempted.
69    Deferred {
70        secs: f64,
71    },
72    Unavailable(String),
73    Disabled,
74    /// The signed-in account or scope changed and the app has not consented. Nothing was
75    /// uploaded, downloaded or written. Call [`SyncEngine::grant_consent`] with `to`.
76    ConsentRequired(ConsentRequest),
77    /// Auth generation changed or sync was switched off mid-pass.
78    Cancelled,
79    Failed(String),
80}
81
82#[derive(Debug, Clone, PartialEq)]
83pub struct PassReport {
84    pub outcome: PassOutcome,
85    pub uploaded: usize,
86    pub downloaded: usize,
87    pub tombstones_applied: usize,
88    pub tombstones_written: usize,
89    pub quarantined: usize,
90    pub blocked_reuploads: usize,
91    /// Seconds until the next pass should run, when one is scheduled.
92    pub retry_after_secs: Option<f64>,
93}
94
95impl PassReport {
96    fn new() -> Self {
97        Self {
98            outcome: PassOutcome::Synced,
99            uploaded: 0,
100            downloaded: 0,
101            tombstones_applied: 0,
102            tombstones_written: 0,
103            quarantined: 0,
104            blocked_reuploads: 0,
105            retry_after_secs: None,
106        }
107    }
108}
109
110/// Cumulative engine counters and current ledger queue sizes, from [`SyncEngine::metrics`].
111/// Queue sizes are the figures that grow resident memory: the ledger keeps one remote
112/// version per record and is serialized in full on every durable write.
113#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
114pub struct SyncMetrics {
115    /// Passes run since the engine was built (including deferred ones).
116    pub passes: u64,
117    /// Passes that ended `Failed` or `Unavailable`.
118    pub failed_passes: u64,
119    /// Wall-clock duration of the most recent pass, in milliseconds.
120    pub last_pass_ms: u64,
121    pub downloaded_total: u64,
122    pub uploaded_total: u64,
123    pub tombstones_applied_total: u64,
124    pub tombstones_written_total: u64,
125    /// Remote versions cached in the ledger (one per known record).
126    pub remote_versions: u64,
127    /// Tombstones the ledger blocks re-import of, forever.
128    pub tombstones: u64,
129    pub pending_uploads: u64,
130    pub pending_tombstones: u64,
131    /// Downloaded records whose local apply was deferred.
132    pub pending_apply: u64,
133}
134
135/// Runtime figures of the engine, computed on call by [`SyncEngine::status`].
136#[derive(Debug, Clone, Copy, PartialEq)]
137pub struct SyncStatus {
138    /// Pass counters and the ledger queue sizes (`pending_*`, `remote_versions`).
139    pub metrics: SyncMetrics,
140    /// Resident memory, CPU time and I/O counters of the whole process (the engine runs in it and
141    /// has no thread of its own), or `None` when the host cannot report them. See
142    /// `rightkit_child_process::ProcessUsage`.
143    #[cfg(feature = "process-usage")]
144    pub process: Option<rightkit_child_process::ProcessUsage>,
145}
146
147#[derive(Debug, Default)]
148struct Counters {
149    passes: u64,
150    failed_passes: u64,
151    last_pass_ms: u64,
152    downloaded: u64,
153    uploaded: u64,
154    tombstones_applied: u64,
155    tombstones_written: u64,
156}
157
158const MAX_CONFLICT_ROUNDS: usize = 4;
159/// Local items fetched per `LocalStore::list_page` call during a pass.
160const LOCAL_PAGE_SIZE: usize = 1000;
161/// Downloaded items decoded and handed to `LocalStore::apply_remote` per call.
162const APPLY_CHUNK: usize = 1000;
163
164enum Resolved {
165    /// Re-saved on top of the server's version.
166    Saved,
167    /// The server version stands (applied locally, tombstoned, or equal).
168    RemoteWon,
169    Failed(String),
170}
171
172enum RemoteState {
173    /// Header of the decoded record; the body is re-decoded from `record` when needed, so
174    /// decoded items are never held for the whole delta.
175    Live(ItemHeader),
176    Tombstone,
177}
178
179struct RemoteEntry {
180    id: String,
181    record_name: String,
182    /// updatedAt for live records, deletedAt for tombstones.
183    version_ms: i64,
184    state: RemoteState,
185    record: RawRecord,
186}
187
188impl RemoteEntry {
189    fn is_tombstone(&self) -> bool {
190        matches!(self.state, RemoteState::Tombstone)
191    }
192    /// Total order for alias reduction: tombstone > live, then newer
193    /// version, then larger record name (deterministic on every device).
194    fn rank(&self) -> (bool, i64, &str) {
195        (
196            self.is_tombstone(),
197            self.version_ms,
198            self.record_name.as_str(),
199        )
200    }
201}
202
203pub struct SyncEngine<M: RecordMapper> {
204    mapper: Arc<M>,
205    backend: Arc<dyn Backend>,
206    auth: Arc<dyn AuthProvider>,
207    local: Arc<dyn LocalStore<M::Item>>,
208    policy: Box<dyn ConflictPolicy<M::Item>>,
209    clock: Arc<dyn Clock>,
210    observer: Arc<dyn SyncObserver>,
211    hooks: Arc<dyn PassHooks>,
212    config: SyncConfig,
213    store: LedgerStore,
214    replay_flags: Vec<String>,
215    counters: Counters,
216    /// Set by `bind_account` for the pass in flight when it stopped on a consent gate.
217    gate: Option<ConsentRequest>,
218    local_page_size: usize,
219    /// Most uploads one pass holds and sends; 0 = no limit.
220    max_uploads: usize,
221}
222
223pub struct SyncEngineBuilder<M: RecordMapper> {
224    mapper: Arc<M>,
225    backend: Arc<dyn Backend>,
226    local: Arc<dyn LocalStore<M::Item>>,
227    storage: Box<dyn LedgerStorage>,
228    auth: Arc<dyn AuthProvider>,
229    policy: Box<dyn ConflictPolicy<M::Item>>,
230    clock: Arc<dyn Clock>,
231    observer: Arc<dyn SyncObserver>,
232    hooks: Arc<dyn PassHooks>,
233    config: SyncConfig,
234    local_page_size: usize,
235    max_uploads: usize,
236}
237
238impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
239    pub fn new(
240        mapper: Arc<M>,
241        backend: Arc<dyn Backend>,
242        local: Arc<dyn LocalStore<M::Item>>,
243        storage: Box<dyn LedgerStorage>,
244    ) -> Self
245    where
246        M::Item: 'static,
247    {
248        Self {
249            mapper,
250            backend,
251            local,
252            storage,
253            auth: Arc::new(AlwaysAvailable),
254            policy: Box::new(LastWriterWins),
255            clock: Arc::new(SystemClock),
256            observer: Arc::new(NullObserver),
257            hooks: Arc::new(NoHooks),
258            config: SyncConfig::default(),
259            local_page_size: LOCAL_PAGE_SIZE,
260            max_uploads: 0,
261        }
262    }
263    /// Items requested per `LocalStore::list_page` call during a pass (default 1000, minimum 1).
264    pub fn local_page_size(mut self, n: usize) -> Self {
265        self.local_page_size = n.max(1);
266        self
267    }
268    /// Most items one pass keeps in memory to upload (default 0 = no limit). The pass keeps the
269    /// `n` that sort first by (origin time, record name), the order uploads already use, and
270    /// leaves the rest for a later pass: they are not marked as published, so the next scan finds
271    /// them again. When any were left, the pass reports the event `uploads_deferred` with their
272    /// count, so the host can run another pass at once. Without a limit a first sync of a large
273    /// store holds every item to publish at the same time.
274    pub fn max_uploads_per_pass(mut self, n: usize) -> Self {
275        self.max_uploads = n;
276        self
277    }
278    pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
279        self.auth = v;
280        self
281    }
282    pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
283        self.policy = v;
284        self
285    }
286    pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
287        self.clock = v;
288        self
289    }
290    pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
291        self.observer = v;
292        self
293    }
294    pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
295        self.hooks = v;
296        self
297    }
298    pub fn config(mut self, v: SyncConfig) -> Self {
299        self.config = v;
300        self
301    }
302    pub fn build(self) -> SyncResult<SyncEngine<M>> {
303        let store = LedgerStore::open(self.storage)?;
304        if store.recovered_from_corruption {
305            self.observer
306                .event("ledger_recovered", "ledger unreadable; full replay");
307        }
308        Ok(SyncEngine {
309            mapper: self.mapper,
310            backend: self.backend,
311            auth: self.auth,
312            local: self.local,
313            policy: self.policy,
314            clock: self.clock,
315            observer: self.observer,
316            hooks: self.hooks,
317            config: self.config,
318            store,
319            replay_flags: Vec::new(),
320            counters: Counters::default(),
321            gate: None,
322            local_page_size: self.local_page_size,
323            max_uploads: self.max_uploads,
324        })
325    }
326}
327
328struct Fetched {
329    records: Vec<RawRecord>,
330    deleted: Vec<String>,
331    token: Option<Vec<u8>>,
332}
333
334impl<M: RecordMapper + 'static> SyncEngine<M>
335where
336    M::Item: 'static,
337{
338    pub fn ledger(&self) -> &crate::ledger::Ledger {
339        self.store.ledger()
340    }
341
342    /// The scope change currently awaiting consent, if any. Durable: it survives a restart and
343    /// is also reported by [`PassOutcome::ConsentRequired`].
344    pub fn consent_request(&self) -> Option<ConsentRequest> {
345        self.store.ledger().pending_consent.clone()
346    }
347
348    /// Record explicit consent to sync into the new account `to` (the `to` of the pending
349    /// [`ConsentRequest`]). Resets the ledger for the new scope (the old account's cursor,
350    /// versions and queued deletions never reach it) and lets the next pass run. Refused when
351    /// nothing is pending, when `to` is not the pending target, or when the signed-in scope has
352    /// since moved on (the next pass then raises a fresh request).
353    pub fn grant_consent(&mut self, to: &str, choice: ConsentChoice) -> SyncResult<()> {
354        let Some(req) = self.store.ledger().pending_consent.clone() else {
355            return Err(SyncError::Transient(
356                "no scope change is awaiting consent".into(),
357            ));
358        };
359        if req.to != to {
360            return Err(SyncError::Transient(
361                "consent names a different account than the one pending".into(),
362            ));
363        }
364        if let Some(current) = self.auth.account_scope() {
365            if current != req.to {
366                return Err(SyncError::Transient(
367                    "the signed-in account changed again; consent is stale".into(),
368                ));
369            }
370        }
371        let withheld: BTreeSet<String> = match choice {
372            ConsentChoice::Merge => BTreeSet::new(),
373            ConsentChoice::StartFresh => {
374                let mut ids = BTreeSet::new();
375                self.scan_local(|i| {
376                    ids.insert(self.mapper.header(&i).id);
377                    Ok(())
378                })?;
379                ids
380            }
381        };
382        self.hooks.on_account_switch(&req.from, &req.to);
383        let scope = req.to.clone();
384        self.store.mutate(|l| {
385            *l = crate::ledger::Ledger::default();
386            l.scope = Some(scope);
387            l.withheld = withheld;
388        })?;
389        self.gate = None;
390        self.observer.event(
391            "consent_granted",
392            match choice {
393                ConsentChoice::Merge => "merge",
394                ConsentChoice::StartFresh => "start_fresh",
395            },
396        );
397        Ok(())
398    }
399
400    /// Visit every local item once, a page at a time through `LocalStore::list_page` (falling
401    /// back to one `all()` call for stores that do not page), so only one page is resident.
402    /// The visitor must not call back into the local store.
403    fn scan_local(&self, mut each: impl FnMut(M::Item) -> SyncResult<()>) -> SyncResult<()> {
404        let size = self.local_page_size.max(1);
405        let mut cursor: Option<String> = None;
406        loop {
407            let (page, last) = match self.local.list_page(cursor.as_deref(), size)? {
408                Some(p) => {
409                    let last = p.len() < size;
410                    (p, last)
411                }
412                None => (self.local.all()?, true),
413            };
414            let next = if last {
415                None
416            } else {
417                page.last().map(|i| self.mapper.header(i).id)
418            };
419            if let (Some(n), Some(c)) = (next.as_ref(), cursor.as_ref()) {
420                if n <= c {
421                    return Err(SyncError::Persistence(
422                        "LocalStore::list_page did not advance past its cursor".into(),
423                    ));
424                }
425            }
426            for item in page {
427                each(item)?;
428            }
429            match next {
430                Some(n) => cursor = Some(n),
431                None => return Ok(()),
432            }
433        }
434    }
435
436    /// Offline queue: record that `id` changed locally. Durable immediately.
437    pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
438        let id = id.to_string();
439        self.store.mutate(|l| {
440            if !l.tombstones.contains_key(&id) {
441                l.pending_uploads.insert(id);
442            }
443        })
444    }
445
446    /// Local deletion: block the id and queue a tombstone write. Durable.
447    pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
448        let now = self.clock.now_ms();
449        let id = id.to_string();
450        self.store.mutate(|l| {
451            l.bury(&id, now);
452            l.pending_tombstones.insert(id);
453        })
454    }
455
456    /// [`SyncEngine::note_deleted`] for many ids with one durable ledger write. The ledger is
457    /// serialized and replaced in full on every write, so a caller with many deletions must not
458    /// call `note_deleted` in a loop. Nothing is written when no id is queued.
459    ///
460    /// An id is in one of three states: (a) not buried; (b) buried and confirmed (in
461    /// `tombstones`, not in `pending_tombstones`: the remote already holds the deletion); (c)
462    /// buried and pending. A state (b) id is a no-op: it is not queued again and its tombstone
463    /// value is not rewritten, so a later withdrawal can never lift a confirmed deletion. The one
464    /// exception is an id in `unconfirmed`, which the caller cannot vouch the remote holds (a
465    /// tombstone kept after its queued write was withdrawn without proof); it is queued again.
466    /// An id that already has a tombstone keeps that tombstone's value; only an id with none is
467    /// buried (now). Returns how many ids were queued.
468    pub fn note_deleted_many(
469        &mut self,
470        ids: &[String],
471        unconfirmed: &BTreeSet<String>,
472    ) -> SyncResult<usize> {
473        let now = self.clock.now_ms();
474        let queue: Vec<String> = {
475            let ledger = self.store.ledger();
476            ids.iter()
477                .filter(|id| {
478                    let confirmed = ledger.tombstones.contains_key(*id)
479                        && !ledger.pending_tombstones.contains(*id);
480                    !confirmed || unconfirmed.contains(*id)
481                })
482                .cloned()
483                .collect()
484        };
485        if queue.is_empty() {
486            return Ok(0);
487        }
488        self.store.mutate(|l| {
489            for id in &queue {
490                if !l.tombstones.contains_key(id) {
491                    l.bury(id, now);
492                }
493                l.pending_tombstones.insert(id.clone());
494            }
495        })?;
496        Ok(queue.len())
497    }
498
499    /// Withdraw queued tombstones the caller can no longer vouch for. Every id of `lift` and
500    /// `keep` that is in `pending_tombstones` leaves the queue, so it is not written to the
501    /// zone. `lift` names ids whose tombstone the caller proves it created (no tombstone existed
502    /// before it queued them): that local tombstone is lifted with the queue entry, so the stale
503    /// deletion neither reaches the remote nor keeps suppressing the record. `keep` names every
504    /// other queued id (prior state unknown): its tombstone stays, so local suppression is not
505    /// lifted; an id in both lists is kept. Ids that are not queued are left alone: a tombstone
506    /// the remote already acknowledged (in `tombstones`, not in `pending_tombstones`) is never
507    /// touched, so a confirmed deletion survives a requeue followed by a withdrawal. One durable
508    /// ledger write; nothing is written when no id is queued. Returns how many ids left the
509    /// queue. A failed write rolls the change back.
510    pub fn withdraw_pending_tombstones(
511        &mut self,
512        lift: &[String],
513        keep: &[String],
514    ) -> SyncResult<usize> {
515        let (queued, lifted): (BTreeSet<String>, BTreeSet<String>) = {
516            let ledger = self.store.ledger();
517            let kept: BTreeSet<&String> = keep.iter().collect();
518            let queued: BTreeSet<String> = lift
519                .iter()
520                .chain(keep.iter())
521                .filter(|id| ledger.pending_tombstones.contains(*id))
522                .cloned()
523                .collect();
524            let lifted: BTreeSet<String> = lift
525                .iter()
526                .filter(|id| queued.contains(*id) && !kept.contains(id))
527                .cloned()
528                .collect();
529            (queued, lifted)
530        };
531        if queued.is_empty() {
532            return Ok(0);
533        }
534        self.store.mutate(|l| {
535            for id in &queued {
536                l.pending_tombstones.remove(id);
537            }
538            for id in &lifted {
539                l.tombstones.remove(id);
540            }
541        })?;
542        Ok(queued.len())
543    }
544
545    /// Replay the whole zone once for `flag` (one-shot repair migrations).
546    pub fn request_full_replay_once(&mut self, flag: &str) {
547        if !self.store.ledger().flags.contains(flag) {
548            self.replay_flags.push(flag.to_string());
549        }
550    }
551
552    /// Runs one sync pass and records it in [`SyncEngine::metrics`].
553    pub fn run_pass(&mut self) -> PassReport {
554        let started = Instant::now();
555        let rep = self.run_pass_once();
556        self.record_pass(&rep, started.elapsed());
557        rep
558    }
559
560    fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
561        let c = &mut self.counters;
562        c.passes += 1;
563        if matches!(
564            rep.outcome,
565            PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
566        ) {
567            c.failed_passes += 1;
568        }
569        c.last_pass_ms = elapsed.as_millis() as u64;
570        c.downloaded += rep.downloaded as u64;
571        c.uploaded += rep.uploaded as u64;
572        c.tombstones_applied += rep.tombstones_applied as u64;
573        c.tombstones_written += rep.tombstones_written as u64;
574    }
575
576    /// Pass counters, ledger queue sizes and process resource usage, read now. No polling thread
577    /// runs for this.
578    pub fn status(&self) -> SyncStatus {
579        SyncStatus {
580            metrics: self.metrics(),
581            #[cfg(feature = "process-usage")]
582            process: rightkit_child_process::process_usage().ok(),
583        }
584    }
585
586    /// Cumulative pass counters and current ledger queue sizes.
587    pub fn metrics(&self) -> SyncMetrics {
588        let l = self.store.ledger();
589        let c = &self.counters;
590        SyncMetrics {
591            passes: c.passes,
592            failed_passes: c.failed_passes,
593            last_pass_ms: c.last_pass_ms,
594            downloaded_total: c.downloaded,
595            uploaded_total: c.uploaded,
596            tombstones_applied_total: c.tombstones_applied,
597            tombstones_written_total: c.tombstones_written,
598            remote_versions: l.remote_versions.len() as u64,
599            tombstones: l.tombstones.len() as u64,
600            pending_uploads: l.pending_uploads.len() as u64,
601            pending_tombstones: l.pending_tombstones.len() as u64,
602            pending_apply: l.pending_apply.len() as u64,
603        }
604    }
605
606    fn run_pass_once(&mut self) -> PassReport {
607        let mut rep = PassReport::new();
608        if !self.auth.enabled() {
609            rep.outcome = PassOutcome::Disabled;
610            return rep;
611        }
612        let now = self.clock.now_ms();
613        if let Some(until) = self.store.ledger().retry_after_ms {
614            if until > now {
615                let secs = (until - now) as f64 / 1000.0;
616                rep.outcome = PassOutcome::Deferred { secs };
617                rep.retry_after_secs = Some(secs);
618                return rep;
619            }
620        }
621        let generation = self.auth.generation();
622        self.gate = None;
623        let result = self.pass_inner(generation, &mut rep);
624        if let Some(req) = self.gate.take() {
625            if result.is_ok() {
626                rep.outcome = PassOutcome::ConsentRequired(req);
627                return rep;
628            }
629        }
630        match result {
631            Ok(pending) => {
632                let result = self.store.mutate(|l| {
633                    if pending.is_none() {
634                        l.consecutive_failures = 0;
635                        l.retry_after_ms = None;
636                    }
637                });
638                match (result, pending) {
639                    (Err(e), _) => self.fail(e, &mut rep),
640                    (Ok(()), None) => rep.outcome = PassOutcome::Synced,
641                    (Ok(()), Some(why)) => {
642                        let secs = self.config.backoff.pending_floor_secs;
643                        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
644                        let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
645                        rep.retry_after_secs = Some(secs);
646                        rep.outcome = PassOutcome::Pending(why);
647                    }
648                }
649            }
650            Err(e) => self.fail(e, &mut rep),
651        }
652        rep
653    }
654
655    fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
656        if e == SyncError::Cancelled {
657            rep.outcome = PassOutcome::Cancelled;
658            return;
659        }
660        self.observer.event("pass_failed", &e.to_string());
661        let failures = self.store.ledger().consecutive_failures.saturating_add(1);
662        let secs = self
663            .config
664            .backoff
665            .delay_secs(failures, e.retry_hint_secs());
666        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
667        let _ = self.store.mutate(|l| {
668            l.consecutive_failures = failures;
669            l.retry_after_ms = Some(until);
670        });
671        rep.retry_after_secs = Some(secs);
672        rep.outcome = match e {
673            SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
674            other => PassOutcome::Failed(other.to_string()),
675        };
676    }
677
678    /// A conditional save lost a race. Read the server record, decide with
679    /// the conflict policy (tombstone always wins) and either re-save on top
680    /// of the server's change tag, take the remote, or keep both unchanged.
681    /// Bounded: a record that keeps changing under us stays pending.
682    fn resolve_conflict(
683        &mut self,
684        header: &ItemHeader,
685        item: &M::Item,
686        name: &str,
687        mut server: Option<RawRecord>,
688        generation: u64,
689        rep: &mut PassReport,
690    ) -> SyncResult<Resolved> {
691        for _ in 0..MAX_CONFLICT_ROUNDS {
692            self.check_current(generation)?;
693            let server_rec = match server.take() {
694                Some(r) => r,
695                None => match self.backend.fetch_record(name)? {
696                    Some(r) => r,
697                    None => return Ok(Resolved::Failed("conflicting record vanished".into())),
698                },
699            };
700            self.observer.event("save_conflict", name);
701            if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
702                self.local.remove(&id)?;
703                self.store.mutate(|l| l.bury(&id, deleted_at))?;
704                rep.tombstones_applied += 1;
705                return Ok(Resolved::RemoteWon);
706            }
707            let remote_item = match self.mapper.decode(&server_rec) {
708                Ok(i) => i,
709                Err(e) => {
710                    rep.quarantined += 1;
711                    return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
712                }
713            };
714            let rh = self.mapper.header(&remote_item);
715            if self
716                .policy
717                .upload_local_over_remote(header, item, &rh, &remote_item)
718            {
719                let mut rec = server_rec;
720                rec.set(field::ID, FieldValue::Text(header.id.clone()));
721                rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
722                rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
723                rec.fields.remove(field::DELETED_AT);
724                self.mapper.encode(item, &mut rec);
725                let outcomes = self
726                    .backend
727                    .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
728                let Some(o) = outcomes.into_iter().next() else {
729                    return Ok(Resolved::Failed("no save outcome".into()));
730                };
731                match o.result {
732                    Ok(()) => return Ok(Resolved::Saved),
733                    Err(SyncError::Conflict) => server = o.server_record,
734                    Err(e) => return Ok(Resolved::Failed(e.to_string())),
735                }
736                continue;
737            }
738            let version = RemoteVersion {
739                record_name: name.to_string(),
740                updated_at_ms: rh.updated_at_ms,
741            };
742            if self.policy.accept_remote_over_local(header, &rh) {
743                self.check_current(generation)?;
744                match self.local.apply_remote(&[remote_item])?.into_iter().next() {
745                    Some(ApplyOutcome::Deferred(_)) => {
746                        let id = header.id.clone();
747                        self.store.mutate(|l| {
748                            l.pending_apply.insert(id, server_rec);
749                        })?;
750                    }
751                    _ => rep.downloaded += 1,
752                }
753            }
754            let id = header.id.clone();
755            self.store.mutate(|l| {
756                l.pending_uploads.remove(&id);
757                l.remote_versions.insert(id, version);
758            })?;
759            return Ok(Resolved::RemoteWon);
760        }
761        Ok(Resolved::Failed("conflict retries exhausted".into()))
762    }
763
764    /// Bind the ledger to the signed-in account scope; reset it when the
765    /// scope changed so one account's cursor, versions and queued deletions
766    /// are never applied to another.
767    ///
768    /// With `require_consent_on_scope_change` a switch from a bound scope is not applied: the
769    /// ledger is left untouched, the request is persisted and `self.gate` is set so the pass
770    /// stops before any I/O.
771    fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
772        self.check_current(generation)?;
773        let Some(scope) = self.auth.account_scope() else {
774            return Ok(());
775        };
776        match self.store.ledger().scope.clone() {
777            Some(bound) if bound == scope => {
778                if self.store.ledger().pending_consent.is_some() {
779                    // The account went back to the bound one: the request is moot.
780                    self.store.mutate(|l| l.pending_consent = None)?;
781                }
782                Ok(())
783            }
784            Some(bound) if self.config.require_consent_on_scope_change => {
785                let req = ConsentRequest {
786                    from: bound,
787                    to: scope,
788                };
789                if self.store.ledger().pending_consent.as_ref() != Some(&req) {
790                    let stored = req.clone();
791                    self.store.mutate(|l| l.pending_consent = Some(stored))?;
792                    self.observer
793                        .event("consent_required", &format!("{} -> {}", req.from, req.to));
794                }
795                self.gate = Some(req);
796                Ok(())
797            }
798            None => {
799                // Ledger from before scoping: keep tombstones (they only
800                // suppress local copies); never import its cursor, versions,
801                // aliases or queued deletions into this account.
802                let dropped = {
803                    let l = self.store.ledger();
804                    l.change_token.is_some()
805                        || !l.remote_versions.is_empty()
806                        || !l.aliases.is_empty()
807                        || !l.pending_tombstones.is_empty()
808                };
809                self.store.mutate(|l| {
810                    l.scope = Some(scope);
811                    l.change_token = None;
812                    l.remote_versions.clear();
813                    l.aliases.clear();
814                    l.pending_tombstones.clear();
815                    l.has_synced = false;
816                })?;
817                if dropped {
818                    self.observer
819                        .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
820                }
821                Ok(())
822            }
823            Some(bound) => {
824                self.hooks.on_account_switch(&bound, &scope);
825                self.store.mutate(|l| {
826                    *l = crate::ledger::Ledger::default();
827                    l.scope = Some(scope.clone());
828                })?;
829                self.observer.event("account_switched", "ledger reset");
830                Ok(())
831            }
832        }
833    }
834
835    fn check_current(&self, generation: u64) -> SyncResult<()> {
836        if self.auth.enabled() && self.auth.generation() == generation {
837            Ok(())
838        } else {
839            Err(SyncError::Cancelled)
840        }
841    }
842
843    /// Returns `Some(reason)` when the pass finished with queued work left.
844    fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
845        self.auth.availability()?;
846        // `availability` refreshes the account identity, which may bump the
847        // generation; the pass belongs to the generation observed after it.
848        let generation = self.auth.generation().max(generation);
849        self.bind_account(generation)?;
850        if self.gate.is_some() {
851            return Ok(None);
852        }
853        self.backend.prepare()?;
854        self.check_current(generation)?;
855
856        // Domain announce-style records go first; failure fails the pass.
857        let mut extras = self.store.ledger().extras.clone();
858        let outbound = self.hooks.outbound(&mut extras);
859        if !outbound.is_empty() {
860            self.check_current(generation)?;
861            self.write_required(&outbound)?;
862        }
863        self.check_current(generation)?;
864        self.store.mutate(|l| l.extras = extras)?;
865
866        // One-shot full replays (repair migrations) reset only the cursor.
867        if !self.replay_flags.is_empty() {
868            self.store.mutate(|l| l.change_token = None)?;
869        }
870
871        let fetched = self.fetch_all(generation)?;
872        let mapper_type = self.mapper.record_type().to_string();
873        let mut entries: Vec<RemoteEntry> = Vec::new();
874        let mut foreign: Vec<RawRecord> = Vec::new();
875        for record in fetched.records {
876            if record.record_type != mapper_type {
877                foreign.push(record);
878                continue;
879            }
880            if let Some((id, deleted_at)) = read_tombstone(&record) {
881                entries.push(RemoteEntry {
882                    id,
883                    record_name: record.record_name.clone(),
884                    version_ms: deleted_at,
885                    state: RemoteState::Tombstone,
886                    record,
887                });
888                continue;
889            }
890            match self.mapper.decode(&record) {
891                Ok(item) => {
892                    let h = self.mapper.header(&item);
893                    entries.push(RemoteEntry {
894                        id: h.id.clone(),
895                        record_name: record.record_name.clone(),
896                        version_ms: h.updated_at_ms,
897                        state: RemoteState::Live(h),
898                        record,
899                    });
900                }
901                Err(why) => {
902                    // A bad record never blocks the page (both platforms).
903                    rep.quarantined += 1;
904                    self.observer.event(
905                        "record_quarantined",
906                        &format!("{}: {why}", record.record_name),
907                    );
908                }
909            }
910        }
911        // Server-side hard deletions of records we know behave as tombstones.
912        for name in &fetched.deleted {
913            let known = self
914                .store
915                .ledger()
916                .aliases
917                .iter()
918                .find(|(_, names)| names.contains(name))
919                .map(|(id, _)| id.clone())
920                .or_else(|| {
921                    self.store
922                        .ledger()
923                        .remote_versions
924                        .contains_key(name)
925                        .then(|| name.clone())
926                });
927            if let Some(id) = known {
928                if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
929                    let mut record = RawRecord::new(&mapper_type, name);
930                    let now = self.clock.now_ms();
931                    write_tombstone(&mut record, &id, now, &[]);
932                    entries.push(RemoteEntry {
933                        id,
934                        record_name: name.clone(),
935                        version_ms: now,
936                        state: RemoteState::Tombstone,
937                        record,
938                    });
939                }
940            }
941        }
942        if rep.quarantined > 0 {
943            let n = rep.quarantined as u64;
944            self.store.mutate(|l| l.quarantined_total += n)?;
945        }
946
947        // Aliases: every record name ever seen for a logical id.
948        self.store.mutate(|l| {
949            for e in &entries {
950                l.aliases
951                    .entry(e.id.clone())
952                    .or_default()
953                    .insert(e.record_name.clone());
954            }
955        })?;
956
957        // Reduce duplicate records of one logical id.
958        let mut canonical: BTreeMap<String, RemoteEntry> = BTreeMap::new();
959        for entry in entries {
960            match canonical.get(&entry.id) {
961                Some(cur) if cur.rank() >= entry.rank() => {}
962                _ => {
963                    canonical.insert(entry.id.clone(), entry);
964                }
965            }
966        }
967
968        // Domain-owned record types (queue requests etc.).
969        if !foreign.is_empty() {
970            let mut extras = self.store.ledger().extras.clone();
971            let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
972            // Responses are durable on the zone BEFORE the extras that record
973            // them are persisted, and a failed write aborts the pass before
974            // the change token can pass the request: the delta replays and
975            // the idempotent hook answers again. Never acknowledge a request
976            // whose response never landed.
977            if !responses.is_empty() {
978                self.check_current(generation)?;
979                if let Err(e) = self.write_required(&responses) {
980                    self.observer.event("foreign_write_failed", &e.to_string());
981                    return Err(e);
982                }
983            }
984            self.check_current(generation)?;
985            self.store.mutate(|l| l.extras = extras)?;
986        }
987
988        // Remote tombstones are authoritative: delete locally, block id.
989        let tombs: Vec<(String, i64)> = canonical
990            .values()
991            .filter(|e| e.is_tombstone())
992            .map(|e| (e.id.clone(), e.version_ms))
993            .collect();
994        for (id, at) in &tombs {
995            self.check_current(generation)?;
996            self.local.remove(id)?;
997            let (id, at) = (id.clone(), *at);
998            self.store.mutate(|l| {
999                l.bury(&id, at);
1000                l.pending_tombstones.remove(&id);
1001            })?;
1002        }
1003        rep.tombstones_applied = tombs.len();
1004        self.check_current(generation)?;
1005
1006        let mut pending: Vec<String> = Vec::new();
1007
1008        // Our own queued deletions.
1009        match self.flush_tombstones(&canonical, generation, rep) {
1010            Ok(true) => {}
1011            Ok(false) => pending.push("tombstones pending".into()),
1012            Err(e) => return Err(e),
1013        }
1014
1015        // ---- Local scan ----
1016        // One paged pass over the local store feeds the inferred deletions, the stale-queue
1017        // check, the upload plan and the download comparison, so at most one page of local
1018        // items is resident; only items that must upload are kept, and only the headers of
1019        // items that also have a live remote entry.
1020        struct Upload<I> {
1021            header: ItemHeader,
1022            item: I,
1023            base: Option<RawRecord>,
1024            is_new: bool,
1025        }
1026        /// An upload with the key uploads are ordered by, so that a bounded queue keeps the
1027        /// same items a full sort would put first.
1028        struct Queued<I> {
1029            key: (i64, String),
1030            upload: Upload<I>,
1031        }
1032        impl<I> PartialEq for Queued<I> {
1033            fn eq(&self, other: &Self) -> bool {
1034                self.key == other.key
1035            }
1036        }
1037        impl<I> Eq for Queued<I> {}
1038        impl<I> PartialOrd for Queued<I> {
1039            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
1040                Some(self.cmp(other))
1041            }
1042        }
1043        impl<I> Ord for Queued<I> {
1044            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
1045                self.key.cmp(&other.key)
1046            }
1047        }
1048        /// At most `cap` uploads (0 = no limit): a max-heap on the key, so the top is the newest
1049        /// kept and is the one a newcomer replaces.
1050        struct UploadQueue<I> {
1051            cap: usize,
1052            heap: std::collections::BinaryHeap<Queued<I>>,
1053            held_back: usize,
1054        }
1055        impl<I> UploadQueue<I> {
1056            fn offer(&mut self, key: (i64, String), upload: Upload<I>) {
1057                if self.cap == 0 || self.heap.len() < self.cap {
1058                    self.heap.push(Queued { key, upload });
1059                    return;
1060                }
1061                self.held_back += 1;
1062                let replaces = self.heap.peek().is_some_and(|top| key < top.key);
1063                if replaces {
1064                    self.heap.pop();
1065                    self.heap.push(Queued { key, upload });
1066                }
1067            }
1068        }
1069        let infer = self.config.infer_local_deletions;
1070        let mut gone_candidates: HashSet<String> = if infer {
1071            let l = self.store.ledger();
1072            l.remote_versions
1073                .keys()
1074                .filter(|id| !l.tombstones.contains_key(*id))
1075                .filter(|id| !l.pending_apply.contains_key(*id))
1076                .cloned()
1077                .collect()
1078        } else {
1079            HashSet::new()
1080        };
1081        let mut queued_unseen: HashSet<String> = self
1082            .store
1083            .ledger()
1084            .pending_uploads
1085            .iter()
1086            .cloned()
1087            .collect();
1088        let mut matched: HashMap<String, ItemHeader> = HashMap::new();
1089        let mut queue: UploadQueue<M::Item> = UploadQueue {
1090            cap: self.max_uploads,
1091            heap: std::collections::BinaryHeap::new(),
1092            held_back: 0,
1093        };
1094        let mut failed_uploads = 0usize;
1095        self.scan_local(|item| {
1096            let header = self.mapper.header(&item);
1097            if infer {
1098                gone_candidates.remove(&header.id);
1099            }
1100            queued_unseen.remove(&header.id);
1101            let canon = canonical.get(&header.id);
1102            if let Some(RemoteEntry {
1103                state: RemoteState::Live(_),
1104                ..
1105            }) = canon
1106            {
1107                matched
1108                    .entry(header.id.clone())
1109                    .or_insert_with(|| header.clone());
1110            }
1111            if self.store.ledger().tombstones.contains_key(&header.id) {
1112                rep.blocked_reuploads += 1;
1113                return Ok(());
1114            }
1115            if self.store.ledger().withheld.contains(&header.id) {
1116                return Ok(());
1117            }
1118            let queued = self.store.ledger().pending_uploads.contains(&header.id);
1119            match canon {
1120                Some(RemoteEntry {
1121                    state: RemoteState::Live(rh),
1122                    record,
1123                    ..
1124                }) => {
1125                    if let Ok(remote_item) = self.mapper.decode(record) {
1126                        if self
1127                            .policy
1128                            .upload_local_over_remote(&header, &item, rh, &remote_item)
1129                        {
1130                            let key = (header.created_at_ms, self.mapper.record_name(&header.id));
1131                            queue.offer(
1132                                key,
1133                                Upload {
1134                                    header,
1135                                    item,
1136                                    base: Some(record.clone()),
1137                                    is_new: false,
1138                                },
1139                            );
1140                        }
1141                    }
1142                }
1143                Some(_) => {}
1144                None => {
1145                    if let Some(known) =
1146                        self.store.ledger().remote_versions.get(&header.id).cloned()
1147                    {
1148                        // Cursor delta did not include it: unchanged remotely.
1149                        if known.updated_at_ms < header.updated_at_ms || queued {
1150                            match self.backend.fetch_record(&known.record_name) {
1151                                Ok(base) => {
1152                                    let key =
1153                                        (header.created_at_ms, self.mapper.record_name(&header.id));
1154                                    queue.offer(
1155                                        key,
1156                                        Upload {
1157                                            header,
1158                                            item,
1159                                            base,
1160                                            is_new: false,
1161                                        },
1162                                    );
1163                                }
1164                                Err(e) => {
1165                                    failed_uploads += 1;
1166                                    self.observer.event("record_fetch_failed", &e.to_string());
1167                                }
1168                            }
1169                        }
1170                    } else {
1171                        let key = (header.created_at_ms, self.mapper.record_name(&header.id));
1172                        queue.offer(
1173                            key,
1174                            Upload {
1175                                header,
1176                                item,
1177                                base: None,
1178                                is_new: true,
1179                            },
1180                        );
1181                    }
1182                }
1183            }
1184            Ok(())
1185        })?;
1186        let held_back_uploads = queue.held_back;
1187        let mut uploads: Vec<Upload<M::Item>> =
1188            queue.heap.into_iter().map(|queued| queued.upload).collect();
1189
1190        // Inferred deletions (desktop): published item vanished locally.
1191        if infer {
1192            let gone: Vec<String> = gone_candidates.into_iter().collect();
1193            if !gone.is_empty() {
1194                let now = self.clock.now_ms();
1195                self.store.mutate(|l| {
1196                    for id in gone {
1197                        l.bury(&id, now);
1198                        l.pending_tombstones.insert(id);
1199                    }
1200                })?;
1201                if !self.flush_tombstones(&canonical, generation, rep)? {
1202                    pending.push("tombstones pending".into());
1203                }
1204            }
1205        }
1206
1207        // ---- Uploads ----
1208        // Drop queued ids whose item no longer exists locally.
1209        let stale: Vec<String> = queued_unseen
1210            .into_iter()
1211            .filter(|id| self.store.ledger().pending_uploads.contains(id))
1212            .collect();
1213        if !stale.is_empty() {
1214            self.store.mutate(|l| {
1215                for id in stale {
1216                    l.pending_uploads.remove(&id);
1217                }
1218            })?;
1219        }
1220        // Oldest origin first, ties by record name: deterministic history.
1221        uploads.sort_by(|a, b| {
1222            (
1223                a.header.created_at_ms,
1224                self.mapper.record_name(&a.header.id),
1225            )
1226                .cmp(&(
1227                    b.header.created_at_ms,
1228                    self.mapper.record_name(&b.header.id),
1229                ))
1230        });
1231        let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
1232        let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
1233        let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
1234        if self.config.ordered_new_publication {
1235            chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
1236            let mut it = existing.into_iter().peekable();
1237            while it.peek().is_some() {
1238                chunks.push((false, it.by_ref().take(size).collect()));
1239            }
1240        } else {
1241            let mut it = new_ones.into_iter().chain(existing).peekable();
1242            while it.peek().is_some() {
1243                chunks.push((false, it.by_ref().take(size).collect()));
1244            }
1245        }
1246
1247        let mut blocked_new = false;
1248        for (ordered_new, chunk) in chunks {
1249            if ordered_new && blocked_new {
1250                continue;
1251            }
1252            self.check_current(generation)?;
1253            let mut records = Vec::with_capacity(chunk.len());
1254            let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
1255            for u in &chunk {
1256                let name = self.mapper.record_name(&u.header.id);
1257                let mut rec = u
1258                    .base
1259                    .clone()
1260                    .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
1261                rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
1262                rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
1263                rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
1264                rec.fields.remove(field::DELETED_AT);
1265                self.mapper.encode(&u.item, &mut rec);
1266                by_name.insert(
1267                    rec.record_name.clone(),
1268                    (u.header.id.clone(), u.header.updated_at_ms),
1269                );
1270                records.push(rec);
1271            }
1272            // Conditional: an edit based on a stale change tag (or an insert of
1273            // a name that now exists) is refused, never overwritten.
1274            let outcomes = self
1275                .backend
1276                .save_records(&records, SavePolicy::IfServerUnchanged)?;
1277            let mut saved: Vec<(String, String, i64)> = Vec::new();
1278            let mut chunk_failed = false;
1279            for o in outcomes {
1280                let Some((id, version)) = by_name.get(&o.record_name) else {
1281                    continue;
1282                };
1283                match o.result {
1284                    Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
1285                    Err(SyncError::Conflict) => {
1286                        let upload = chunk
1287                            .iter()
1288                            .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
1289                            .expect("outcome names an upload of this chunk");
1290                        let (header, item) = (upload.header.clone(), upload.item.clone());
1291                        match self.resolve_conflict(
1292                            &header,
1293                            &item,
1294                            &o.record_name,
1295                            o.server_record,
1296                            generation,
1297                            rep,
1298                        )? {
1299                            Resolved::Saved => {
1300                                saved.push((id.clone(), o.record_name.clone(), *version))
1301                            }
1302                            Resolved::RemoteWon => {}
1303                            Resolved::Failed(why) => {
1304                                chunk_failed = true;
1305                                failed_uploads += 1;
1306                                self.observer.event(
1307                                    "record_save_failed",
1308                                    &format!("{}: {why}", o.record_name),
1309                                );
1310                            }
1311                        }
1312                    }
1313                    Err(e) => {
1314                        chunk_failed = true;
1315                        failed_uploads += 1;
1316                        self.observer
1317                            .event("record_save_failed", &format!("{}: {e}", o.record_name));
1318                    }
1319                }
1320            }
1321            if ordered_new && chunk_failed {
1322                blocked_new = true;
1323            }
1324            // Persist progress per chunk: a crash resumes from here.
1325            rep.uploaded += saved.len();
1326            self.store.mutate(|l| {
1327                for (id, name, version) in saved {
1328                    l.pending_uploads.remove(&id);
1329                    l.aliases
1330                        .entry(id.clone())
1331                        .or_default()
1332                        .insert(name.clone());
1333                    l.remote_versions.insert(
1334                        id,
1335                        RemoteVersion {
1336                            record_name: name,
1337                            updated_at_ms: version,
1338                        },
1339                    );
1340                }
1341            })?;
1342        }
1343        if failed_uploads > 0 {
1344            pending.push(format!("{failed_uploads} upload(s) pending"));
1345        }
1346        if held_back_uploads > 0 {
1347            self.observer
1348                .event("uploads_deferred", &held_back_uploads.to_string());
1349        }
1350
1351        // ---- Downloads ----
1352        // Plan first with ids and sort keys only (the global (created, id) order is kept), then
1353        // decode and apply a bounded chunk at a time so item bodies are never all resident.
1354        enum Source {
1355            Remote,
1356            Stash(RawRecord),
1357        }
1358        struct Planned {
1359            created: i64,
1360            id: String,
1361            source: Source,
1362        }
1363        let stash_ids: HashSet<String> =
1364            self.store.ledger().pending_apply.keys().cloned().collect();
1365        let mut taken_stash_ids: HashSet<String> = HashSet::new();
1366        let mut plan: Vec<Planned> = Vec::new();
1367        for entry in canonical.values() {
1368            let RemoteState::Live(rh) = &entry.state else {
1369                continue;
1370            };
1371            if self.store.ledger().tombstones.contains_key(&entry.id) {
1372                continue;
1373            }
1374            let take = match matched.get(&entry.id) {
1375                None => true,
1376                Some(lh) => self.policy.accept_remote_over_local(lh, rh),
1377            };
1378            if take {
1379                if stash_ids.contains(&entry.id) {
1380                    taken_stash_ids.insert(entry.id.clone());
1381                }
1382                plan.push(Planned {
1383                    created: rh.created_at_ms,
1384                    id: entry.id.clone(),
1385                    source: Source::Remote,
1386                });
1387            }
1388        }
1389        drop(matched);
1390        // Retry records whose local apply was deferred earlier.
1391        let stashed: Vec<(String, RawRecord)> = self
1392            .store
1393            .ledger()
1394            .pending_apply
1395            .iter()
1396            .map(|(k, v)| (k.clone(), v.clone()))
1397            .collect();
1398        for (id, record) in stashed {
1399            if taken_stash_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1400                continue;
1401            }
1402            if let Ok(item) = self.mapper.decode(&record) {
1403                let h = self.mapper.header(&item);
1404                plan.push(Planned {
1405                    created: h.created_at_ms,
1406                    id: h.id,
1407                    source: Source::Stash(record),
1408                });
1409            }
1410        }
1411        plan.sort_by(|a, b| (a.created, a.id.as_str()).cmp(&(b.created, b.id.as_str())));
1412        let mut deferred = 0usize;
1413        for chunk in plan.chunks(APPLY_CHUNK) {
1414            self.check_current(generation)?;
1415            let mut items: Vec<M::Item> = Vec::with_capacity(chunk.len());
1416            let mut decoded_at: Vec<usize> = Vec::with_capacity(chunk.len());
1417            for (i, p) in chunk.iter().enumerate() {
1418                let decoded = match &p.source {
1419                    Source::Remote => canonical.get(&p.id).map(|e| self.mapper.decode(&e.record)),
1420                    Source::Stash(rec) => Some(self.mapper.decode(rec)),
1421                };
1422                if let Some(Ok(item)) = decoded {
1423                    items.push(item);
1424                    decoded_at.push(i);
1425                }
1426            }
1427            if items.is_empty() {
1428                continue;
1429            }
1430            let outcomes = self.local.apply_remote(&items)?;
1431            if outcomes.len() != items.len() {
1432                return Err(SyncError::Persistence(
1433                    "apply_remote returned wrong outcome count".into(),
1434                ));
1435            }
1436            drop(items);
1437            let mut applied_ids: Vec<String> = Vec::new();
1438            let mut stash: Vec<(String, RawRecord)> = Vec::new();
1439            for (i, out) in decoded_at.into_iter().zip(outcomes) {
1440                let p = &chunk[i];
1441                match out {
1442                    ApplyOutcome::Applied => {
1443                        rep.downloaded += 1;
1444                        applied_ids.push(p.id.clone());
1445                    }
1446                    ApplyOutcome::Deferred(why) => {
1447                        deferred += 1;
1448                        self.observer
1449                            .event("apply_deferred", &format!("{}: {why}", p.id));
1450                        let record = match &p.source {
1451                            Source::Remote => canonical.get(&p.id).map(|e| e.record.clone()),
1452                            Source::Stash(rec) => Some(rec.clone()),
1453                        };
1454                        if let Some(record) = record {
1455                            stash.push((p.id.clone(), record));
1456                        }
1457                    }
1458                }
1459            }
1460            let touches_ledger = !stash.is_empty()
1461                || applied_ids
1462                    .iter()
1463                    .any(|id| self.store.ledger().pending_apply.contains_key(id));
1464            if touches_ledger {
1465                self.store.mutate(|l| {
1466                    for id in applied_ids {
1467                        l.pending_apply.remove(&id);
1468                    }
1469                    for (id, rec) in stash {
1470                        l.pending_apply.insert(id, rec);
1471                    }
1472                })?;
1473            }
1474        }
1475        drop(plan);
1476        if deferred > 0 {
1477            pending.push(format!("{deferred} download(s) deferred"));
1478        }
1479
1480        // Remote versions + advance the token: only now, after everything
1481        // above applied (or was durably queued).
1482        // Final guard: a pass whose account/enable generation changed during
1483        // I/O must not advance the cursor or mark the ledger synced.
1484        self.check_current(generation)?;
1485        let flags = std::mem::take(&mut self.replay_flags);
1486        let token = fetched.token;
1487        self.store.mutate(|l| {
1488            for e in canonical.values() {
1489                // A local upload this pass may already hold a newer version.
1490                let keep = l
1491                    .remote_versions
1492                    .get(&e.id)
1493                    .is_some_and(|k| k.updated_at_ms > e.version_ms);
1494                if !keep {
1495                    l.remote_versions.insert(
1496                        e.id.clone(),
1497                        RemoteVersion {
1498                            record_name: e.record_name.clone(),
1499                            updated_at_ms: e.version_ms,
1500                        },
1501                    );
1502                }
1503            }
1504            if token.is_some() {
1505                l.change_token = token;
1506            }
1507            l.has_synced = true;
1508            for f in flags {
1509                l.flags.insert(f);
1510            }
1511        })?;
1512        if rep.blocked_reuploads > 0 {
1513            self.observer
1514                .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1515        }
1516        Ok(if pending.is_empty() {
1517            None
1518        } else {
1519            Some(pending.join("; "))
1520        })
1521    }
1522
1523    /// Fetch every page; reset the token once on expiry and re-prepare the
1524    /// zone once if it vanished.
1525    fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1526        let mut reset_token = false;
1527        let mut recreated = false;
1528        'restart: loop {
1529            let mut token = self.store.ledger().change_token.clone();
1530            let mut out = Fetched {
1531                records: Vec::new(),
1532                deleted: Vec::new(),
1533                token: None,
1534            };
1535            loop {
1536                self.check_current(generation)?;
1537                match self.backend.fetch_changes(token.as_deref()) {
1538                    Ok(page) => {
1539                        out.records.extend(page.records);
1540                        out.deleted.extend(page.deleted_record_names);
1541                        if page.new_token.is_some() {
1542                            token = page.new_token.clone();
1543                            out.token = page.new_token;
1544                        }
1545                        if !page.more_coming {
1546                            return Ok(out);
1547                        }
1548                    }
1549                    Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1550                        reset_token = true;
1551                        self.observer
1552                            .event("token_reset", "change token expired; full replay");
1553                        self.store.mutate(|l| l.change_token = None)?;
1554                        continue 'restart;
1555                    }
1556                    Err(SyncError::ZoneMissing) if !recreated => {
1557                        recreated = true;
1558                        self.observer
1559                            .event("zone_recreated", "zone missing; re-preparing");
1560                        // Every remote record is gone: what the ledger believes is
1561                        // published no longer exists, so local items must
1562                        // re-publish (donor: acknowledgements are invalidated).
1563                        self.store.mutate(|l| {
1564                            l.change_token = None;
1565                            l.remote_versions.clear();
1566                            l.aliases.clear();
1567                        })?;
1568                        self.backend.prepare()?;
1569                        continue 'restart;
1570                    }
1571                    Err(e) => return Err(e),
1572                }
1573            }
1574        }
1575    }
1576
1577    /// Write queued tombstones to every alias record. Returns true when the
1578    /// queue drained.
1579    fn flush_tombstones(
1580        &mut self,
1581        canonical: &BTreeMap<String, RemoteEntry>,
1582        generation: u64,
1583        rep: &mut PassReport,
1584    ) -> SyncResult<bool> {
1585        let ids: Vec<String> = self
1586            .store
1587            .ledger()
1588            .pending_tombstones
1589            .iter()
1590            .cloned()
1591            .collect();
1592        if ids.is_empty() {
1593            return Ok(true);
1594        }
1595        struct Write {
1596            id: String,
1597            record: RawRecord,
1598        }
1599        let mut writes: Vec<Write> = Vec::new();
1600        let mut never_remote: Vec<String> = Vec::new();
1601        for id in &ids {
1602            let mut names: BTreeSet<String> = self
1603                .store
1604                .ledger()
1605                .aliases
1606                .get(id)
1607                .cloned()
1608                .unwrap_or_default();
1609            let known =
1610                self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1611            if names.is_empty() && !known {
1612                never_remote.push(id.clone());
1613                continue;
1614            }
1615            names.insert(self.mapper.record_name(id));
1616            let at = self
1617                .store
1618                .ledger()
1619                .tombstones
1620                .get(id)
1621                .copied()
1622                .unwrap_or_else(|| self.clock.now_ms());
1623            for name in names {
1624                let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1625                    Some(e) => e.record.clone(),
1626                    None => self
1627                        .backend
1628                        .fetch_record(&name)
1629                        .ok()
1630                        .flatten()
1631                        .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1632                };
1633                write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1634                writes.push(Write {
1635                    id: id.clone(),
1636                    record,
1637                });
1638            }
1639        }
1640        if !never_remote.is_empty() {
1641            // Created and deleted before ever reaching the zone: nothing to write.
1642            self.store.mutate(|l| {
1643                for id in never_remote {
1644                    l.pending_tombstones.remove(&id);
1645                }
1646            })?;
1647        }
1648        let mut failed: BTreeSet<String> = BTreeSet::new();
1649        let size = self
1650            .config
1651            .tombstone_batch
1652            .min(self.backend.max_batch())
1653            .max(1);
1654        for chunk in writes.chunks(size) {
1655            self.check_current(generation)?;
1656            let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1657            let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1658            for w in chunk {
1659                let ok = outcomes
1660                    .iter()
1661                    .find(|o| o.record_name == w.record.record_name)
1662                    .is_some_and(|o| o.result.is_ok());
1663                if !ok {
1664                    failed.insert(w.id.clone());
1665                    self.observer
1666                        .event("tombstone_save_failed", &w.record.record_name);
1667                }
1668            }
1669        }
1670        let acked: Vec<String> = writes
1671            .iter()
1672            .map(|w| w.id.clone())
1673            .collect::<BTreeSet<_>>()
1674            .into_iter()
1675            .filter(|id| !failed.contains(id))
1676            .collect();
1677        rep.tombstones_written += acked.len();
1678        self.store.mutate(|l| {
1679            for id in &acked {
1680                l.pending_tombstones.remove(id);
1681            }
1682        })?;
1683        Ok(failed.is_empty())
1684    }
1685
1686    /// Save records whose failure must fail the pass.
1687    fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1688        for chunk in records.chunks(self.backend.max_batch().max(1)) {
1689            for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1690                o.result?;
1691            }
1692        }
1693        Ok(())
1694    }
1695}