Skip to main content

rightkit_sync/
engine.rs

1//! The sync pass. One `run_pass` mirrors the iOS `sync()` actor method and the
2//! desktop `Coordinator::sync_pass`; see API.md for the shared contract.
3
4use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use crate::backoff::BackoffPolicy;
9use crate::error::{SyncError, SyncResult};
10use crate::ledger::{ConsentRequest, LedgerStore, RemoteVersion};
11use crate::record::{field, read_tombstone, write_tombstone, FieldValue, RawRecord};
12use crate::storage::LedgerStorage;
13use crate::traits::*;
14
15#[derive(Debug, Clone)]
16pub struct SyncConfig {
17    /// Upload chunk size for edits to records that already exist remotely.
18    pub batch_size: usize,
19    /// Tombstone write chunk size.
20    pub tombstone_batch: usize,
21    /// Publish never-seen items one record per save, oldest first, and stop
22    /// new publication for the pass after the first failure (preserves
23    /// history order: CloudKit gives no ordering inside one operation).
24    pub ordered_new_publication: bool,
25    /// Treat a known-published item that vanished from the local store as a
26    /// local deletion (desktop). Apps with explicit `note_deleted` calls
27    /// (iOS) leave this off.
28    pub infer_local_deletions: bool,
29    /// When the signed-in account or scope differs from the one the ledger is bound to, pause
30    /// instead of resetting the ledger and publishing local history into the new account. The
31    /// pass reports [`PassOutcome::ConsentRequired`] and uploads, downloads and writes nothing
32    /// until the app calls [`SyncEngine::grant_consent`]. Default `false` (the earlier silent
33    /// reset), so existing apps keep their behaviour.
34    pub require_consent_on_scope_change: bool,
35    pub backoff: BackoffPolicy,
36}
37
38/// The app's answer to a [`PassOutcome::ConsentRequired`] pause.
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub enum ConsentChoice {
41    /// Publish this device's local items into the new account (and download its content).
42    Merge,
43    /// Keep this device's existing local items local: they are never published to the new
44    /// account. The new account's content is still downloaded; items created or received after
45    /// consent sync normally. Nothing local is deleted.
46    StartFresh,
47}
48
49impl Default for SyncConfig {
50    fn default() -> Self {
51        Self {
52            batch_size: 200,
53            tombstone_batch: 50,
54            ordered_new_publication: true,
55            infer_local_deletions: false,
56            require_consent_on_scope_change: false,
57            backoff: BackoffPolicy::default(),
58        }
59    }
60}
61
62#[derive(Debug, Clone, PartialEq)]
63pub enum PassOutcome {
64    /// Everything applied; token advanced; no pending work.
65    Synced,
66    /// Token advanced but some uploads/tombstones/applies remain queued.
67    Pending(String),
68    /// Backoff window still open; nothing was attempted.
69    Deferred {
70        secs: f64,
71    },
72    Unavailable(String),
73    Disabled,
74    /// The signed-in account or scope changed and the app has not consented. Nothing was
75    /// uploaded, downloaded or written. Call [`SyncEngine::grant_consent`] with `to`.
76    ConsentRequired(ConsentRequest),
77    /// Auth generation changed or sync was switched off mid-pass.
78    Cancelled,
79    Failed(String),
80}
81
82#[derive(Debug, Clone, PartialEq)]
83pub struct PassReport {
84    pub outcome: PassOutcome,
85    pub uploaded: usize,
86    pub downloaded: usize,
87    pub tombstones_applied: usize,
88    pub tombstones_written: usize,
89    pub quarantined: usize,
90    pub blocked_reuploads: usize,
91    /// Seconds until the next pass should run, when one is scheduled.
92    pub retry_after_secs: Option<f64>,
93}
94
95impl PassReport {
96    fn new() -> Self {
97        Self {
98            outcome: PassOutcome::Synced,
99            uploaded: 0,
100            downloaded: 0,
101            tombstones_applied: 0,
102            tombstones_written: 0,
103            quarantined: 0,
104            blocked_reuploads: 0,
105            retry_after_secs: None,
106        }
107    }
108}
109
110/// Cumulative engine counters and current ledger queue sizes, from [`SyncEngine::metrics`].
111/// Queue sizes are the figures that grow resident memory: the ledger keeps one remote
112/// version per record and is serialized in full on every durable write.
113#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
114pub struct SyncMetrics {
115    /// Passes run since the engine was built (including deferred ones).
116    pub passes: u64,
117    /// Passes that ended `Failed` or `Unavailable`.
118    pub failed_passes: u64,
119    /// Wall-clock duration of the most recent pass, in milliseconds.
120    pub last_pass_ms: u64,
121    pub downloaded_total: u64,
122    pub uploaded_total: u64,
123    pub tombstones_applied_total: u64,
124    pub tombstones_written_total: u64,
125    /// Remote versions cached in the ledger (one per known record).
126    pub remote_versions: u64,
127    /// Tombstones the ledger blocks re-import of, forever.
128    pub tombstones: u64,
129    pub pending_uploads: u64,
130    pub pending_tombstones: u64,
131    /// Downloaded records whose local apply was deferred.
132    pub pending_apply: u64,
133}
134
135/// Runtime figures of the engine, computed on call by [`SyncEngine::status`].
136#[derive(Debug, Clone, Copy, PartialEq)]
137pub struct SyncStatus {
138    /// Pass counters and the ledger queue sizes (`pending_*`, `remote_versions`).
139    pub metrics: SyncMetrics,
140    /// Resident memory, CPU time and I/O counters of the whole process (the engine runs in it and
141    /// has no thread of its own), or `None` when the host cannot report them. See
142    /// `rightkit_child_process::ProcessUsage`.
143    #[cfg(feature = "process-usage")]
144    pub process: Option<rightkit_child_process::ProcessUsage>,
145}
146
147#[derive(Debug, Default)]
148struct Counters {
149    passes: u64,
150    failed_passes: u64,
151    last_pass_ms: u64,
152    downloaded: u64,
153    uploaded: u64,
154    tombstones_applied: u64,
155    tombstones_written: u64,
156}
157
158const MAX_CONFLICT_ROUNDS: usize = 4;
159/// Local items fetched per `LocalStore::list_page` call during a pass.
160const LOCAL_PAGE_SIZE: usize = 1000;
161/// Downloaded items decoded and handed to `LocalStore::apply_remote` per call.
162const APPLY_CHUNK: usize = 1000;
163
164enum Resolved {
165    /// Re-saved on top of the server's version.
166    Saved,
167    /// The server version stands (applied locally, tombstoned, or equal).
168    RemoteWon,
169    Failed(String),
170}
171
172enum RemoteState {
173    /// Header of the decoded record; the body is re-decoded from `record` when needed, so
174    /// decoded items are never held for the whole delta.
175    Live(ItemHeader),
176    Tombstone,
177}
178
179struct RemoteEntry {
180    id: String,
181    record_name: String,
182    /// updatedAt for live records, deletedAt for tombstones.
183    version_ms: i64,
184    state: RemoteState,
185    record: RawRecord,
186}
187
188impl RemoteEntry {
189    fn is_tombstone(&self) -> bool {
190        matches!(self.state, RemoteState::Tombstone)
191    }
192    /// Total order for alias reduction: tombstone > live, then newer
193    /// version, then larger record name (deterministic on every device).
194    fn rank(&self) -> (bool, i64, &str) {
195        (
196            self.is_tombstone(),
197            self.version_ms,
198            self.record_name.as_str(),
199        )
200    }
201}
202
203pub struct SyncEngine<M: RecordMapper> {
204    mapper: Arc<M>,
205    backend: Arc<dyn Backend>,
206    auth: Arc<dyn AuthProvider>,
207    local: Arc<dyn LocalStore<M::Item>>,
208    policy: Box<dyn ConflictPolicy<M::Item>>,
209    clock: Arc<dyn Clock>,
210    observer: Arc<dyn SyncObserver>,
211    hooks: Arc<dyn PassHooks>,
212    config: SyncConfig,
213    store: LedgerStore,
214    replay_flags: Vec<String>,
215    counters: Counters,
216    /// Set by `bind_account` for the pass in flight when it stopped on a consent gate.
217    gate: Option<ConsentRequest>,
218    local_page_size: usize,
219}
220
221pub struct SyncEngineBuilder<M: RecordMapper> {
222    mapper: Arc<M>,
223    backend: Arc<dyn Backend>,
224    local: Arc<dyn LocalStore<M::Item>>,
225    storage: Box<dyn LedgerStorage>,
226    auth: Arc<dyn AuthProvider>,
227    policy: Box<dyn ConflictPolicy<M::Item>>,
228    clock: Arc<dyn Clock>,
229    observer: Arc<dyn SyncObserver>,
230    hooks: Arc<dyn PassHooks>,
231    config: SyncConfig,
232    local_page_size: usize,
233}
234
235impl<M: RecordMapper + 'static> SyncEngineBuilder<M> {
236    pub fn new(
237        mapper: Arc<M>,
238        backend: Arc<dyn Backend>,
239        local: Arc<dyn LocalStore<M::Item>>,
240        storage: Box<dyn LedgerStorage>,
241    ) -> Self
242    where
243        M::Item: 'static,
244    {
245        Self {
246            mapper,
247            backend,
248            local,
249            storage,
250            auth: Arc::new(AlwaysAvailable),
251            policy: Box::new(LastWriterWins),
252            clock: Arc::new(SystemClock),
253            observer: Arc::new(NullObserver),
254            hooks: Arc::new(NoHooks),
255            config: SyncConfig::default(),
256            local_page_size: LOCAL_PAGE_SIZE,
257        }
258    }
259    /// Items requested per `LocalStore::list_page` call during a pass (default 1000, minimum 1).
260    pub fn local_page_size(mut self, n: usize) -> Self {
261        self.local_page_size = n.max(1);
262        self
263    }
264    pub fn auth(mut self, v: Arc<dyn AuthProvider>) -> Self {
265        self.auth = v;
266        self
267    }
268    pub fn policy(mut self, v: Box<dyn ConflictPolicy<M::Item>>) -> Self {
269        self.policy = v;
270        self
271    }
272    pub fn clock(mut self, v: Arc<dyn Clock>) -> Self {
273        self.clock = v;
274        self
275    }
276    pub fn observer(mut self, v: Arc<dyn SyncObserver>) -> Self {
277        self.observer = v;
278        self
279    }
280    pub fn hooks(mut self, v: Arc<dyn PassHooks>) -> Self {
281        self.hooks = v;
282        self
283    }
284    pub fn config(mut self, v: SyncConfig) -> Self {
285        self.config = v;
286        self
287    }
288    pub fn build(self) -> SyncResult<SyncEngine<M>> {
289        let store = LedgerStore::open(self.storage)?;
290        if store.recovered_from_corruption {
291            self.observer
292                .event("ledger_recovered", "ledger unreadable; full replay");
293        }
294        Ok(SyncEngine {
295            mapper: self.mapper,
296            backend: self.backend,
297            auth: self.auth,
298            local: self.local,
299            policy: self.policy,
300            clock: self.clock,
301            observer: self.observer,
302            hooks: self.hooks,
303            config: self.config,
304            store,
305            replay_flags: Vec::new(),
306            counters: Counters::default(),
307            gate: None,
308            local_page_size: self.local_page_size,
309        })
310    }
311}
312
313struct Fetched {
314    records: Vec<RawRecord>,
315    deleted: Vec<String>,
316    token: Option<Vec<u8>>,
317}
318
319impl<M: RecordMapper + 'static> SyncEngine<M>
320where
321    M::Item: 'static,
322{
323    pub fn ledger(&self) -> &crate::ledger::Ledger {
324        self.store.ledger()
325    }
326
327    /// The scope change currently awaiting consent, if any. Durable: it survives a restart and
328    /// is also reported by [`PassOutcome::ConsentRequired`].
329    pub fn consent_request(&self) -> Option<ConsentRequest> {
330        self.store.ledger().pending_consent.clone()
331    }
332
333    /// Record explicit consent to sync into the new account `to` (the `to` of the pending
334    /// [`ConsentRequest`]). Resets the ledger for the new scope (the old account's cursor,
335    /// versions and queued deletions never reach it) and lets the next pass run. Refused when
336    /// nothing is pending, when `to` is not the pending target, or when the signed-in scope has
337    /// since moved on (the next pass then raises a fresh request).
338    pub fn grant_consent(&mut self, to: &str, choice: ConsentChoice) -> SyncResult<()> {
339        let Some(req) = self.store.ledger().pending_consent.clone() else {
340            return Err(SyncError::Transient(
341                "no scope change is awaiting consent".into(),
342            ));
343        };
344        if req.to != to {
345            return Err(SyncError::Transient(
346                "consent names a different account than the one pending".into(),
347            ));
348        }
349        if let Some(current) = self.auth.account_scope() {
350            if current != req.to {
351                return Err(SyncError::Transient(
352                    "the signed-in account changed again; consent is stale".into(),
353                ));
354            }
355        }
356        let withheld: BTreeSet<String> = match choice {
357            ConsentChoice::Merge => BTreeSet::new(),
358            ConsentChoice::StartFresh => {
359                let mut ids = BTreeSet::new();
360                self.scan_local(|i| {
361                    ids.insert(self.mapper.header(&i).id);
362                    Ok(())
363                })?;
364                ids
365            }
366        };
367        self.hooks.on_account_switch(&req.from, &req.to);
368        let scope = req.to.clone();
369        self.store.mutate(|l| {
370            *l = crate::ledger::Ledger::default();
371            l.scope = Some(scope);
372            l.withheld = withheld;
373        })?;
374        self.gate = None;
375        self.observer.event(
376            "consent_granted",
377            match choice {
378                ConsentChoice::Merge => "merge",
379                ConsentChoice::StartFresh => "start_fresh",
380            },
381        );
382        Ok(())
383    }
384
385    /// Visit every local item once, a page at a time through `LocalStore::list_page` (falling
386    /// back to one `all()` call for stores that do not page), so only one page is resident.
387    /// The visitor must not call back into the local store.
388    fn scan_local(&self, mut each: impl FnMut(M::Item) -> SyncResult<()>) -> SyncResult<()> {
389        let size = self.local_page_size.max(1);
390        let mut cursor: Option<String> = None;
391        loop {
392            let (page, last) = match self.local.list_page(cursor.as_deref(), size)? {
393                Some(p) => {
394                    let last = p.len() < size;
395                    (p, last)
396                }
397                None => (self.local.all()?, true),
398            };
399            let next = if last {
400                None
401            } else {
402                page.last().map(|i| self.mapper.header(i).id)
403            };
404            if let (Some(n), Some(c)) = (next.as_ref(), cursor.as_ref()) {
405                if n <= c {
406                    return Err(SyncError::Persistence(
407                        "LocalStore::list_page did not advance past its cursor".into(),
408                    ));
409                }
410            }
411            for item in page {
412                each(item)?;
413            }
414            match next {
415                Some(n) => cursor = Some(n),
416                None => return Ok(()),
417            }
418        }
419    }
420
421    /// Offline queue: record that `id` changed locally. Durable immediately.
422    pub fn note_changed(&mut self, id: &str) -> SyncResult<()> {
423        let id = id.to_string();
424        self.store.mutate(|l| {
425            if !l.tombstones.contains_key(&id) {
426                l.pending_uploads.insert(id);
427            }
428        })
429    }
430
431    /// Local deletion: block the id and queue a tombstone write. Durable.
432    pub fn note_deleted(&mut self, id: &str) -> SyncResult<()> {
433        let now = self.clock.now_ms();
434        let id = id.to_string();
435        self.store.mutate(|l| {
436            l.bury(&id, now);
437            l.pending_tombstones.insert(id);
438        })
439    }
440
441    /// Replay the whole zone once for `flag` (one-shot repair migrations).
442    pub fn request_full_replay_once(&mut self, flag: &str) {
443        if !self.store.ledger().flags.contains(flag) {
444            self.replay_flags.push(flag.to_string());
445        }
446    }
447
448    /// Runs one sync pass and records it in [`SyncEngine::metrics`].
449    pub fn run_pass(&mut self) -> PassReport {
450        let started = Instant::now();
451        let rep = self.run_pass_once();
452        self.record_pass(&rep, started.elapsed());
453        rep
454    }
455
456    fn record_pass(&mut self, rep: &PassReport, elapsed: Duration) {
457        let c = &mut self.counters;
458        c.passes += 1;
459        if matches!(
460            rep.outcome,
461            PassOutcome::Failed(_) | PassOutcome::Unavailable(_)
462        ) {
463            c.failed_passes += 1;
464        }
465        c.last_pass_ms = elapsed.as_millis() as u64;
466        c.downloaded += rep.downloaded as u64;
467        c.uploaded += rep.uploaded as u64;
468        c.tombstones_applied += rep.tombstones_applied as u64;
469        c.tombstones_written += rep.tombstones_written as u64;
470    }
471
472    /// Pass counters, ledger queue sizes and process resource usage, read now. No polling thread
473    /// runs for this.
474    pub fn status(&self) -> SyncStatus {
475        SyncStatus {
476            metrics: self.metrics(),
477            #[cfg(feature = "process-usage")]
478            process: rightkit_child_process::process_usage().ok(),
479        }
480    }
481
482    /// Cumulative pass counters and current ledger queue sizes.
483    pub fn metrics(&self) -> SyncMetrics {
484        let l = self.store.ledger();
485        let c = &self.counters;
486        SyncMetrics {
487            passes: c.passes,
488            failed_passes: c.failed_passes,
489            last_pass_ms: c.last_pass_ms,
490            downloaded_total: c.downloaded,
491            uploaded_total: c.uploaded,
492            tombstones_applied_total: c.tombstones_applied,
493            tombstones_written_total: c.tombstones_written,
494            remote_versions: l.remote_versions.len() as u64,
495            tombstones: l.tombstones.len() as u64,
496            pending_uploads: l.pending_uploads.len() as u64,
497            pending_tombstones: l.pending_tombstones.len() as u64,
498            pending_apply: l.pending_apply.len() as u64,
499        }
500    }
501
502    fn run_pass_once(&mut self) -> PassReport {
503        let mut rep = PassReport::new();
504        if !self.auth.enabled() {
505            rep.outcome = PassOutcome::Disabled;
506            return rep;
507        }
508        let now = self.clock.now_ms();
509        if let Some(until) = self.store.ledger().retry_after_ms {
510            if until > now {
511                let secs = (until - now) as f64 / 1000.0;
512                rep.outcome = PassOutcome::Deferred { secs };
513                rep.retry_after_secs = Some(secs);
514                return rep;
515            }
516        }
517        let generation = self.auth.generation();
518        self.gate = None;
519        let result = self.pass_inner(generation, &mut rep);
520        if let Some(req) = self.gate.take() {
521            if result.is_ok() {
522                rep.outcome = PassOutcome::ConsentRequired(req);
523                return rep;
524            }
525        }
526        match result {
527            Ok(pending) => {
528                let result = self.store.mutate(|l| {
529                    if pending.is_none() {
530                        l.consecutive_failures = 0;
531                        l.retry_after_ms = None;
532                    }
533                });
534                match (result, pending) {
535                    (Err(e), _) => self.fail(e, &mut rep),
536                    (Ok(()), None) => rep.outcome = PassOutcome::Synced,
537                    (Ok(()), Some(why)) => {
538                        let secs = self.config.backoff.pending_floor_secs;
539                        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
540                        let _ = self.store.mutate(|l| l.retry_after_ms = Some(until));
541                        rep.retry_after_secs = Some(secs);
542                        rep.outcome = PassOutcome::Pending(why);
543                    }
544                }
545            }
546            Err(e) => self.fail(e, &mut rep),
547        }
548        rep
549    }
550
551    fn fail(&mut self, e: SyncError, rep: &mut PassReport) {
552        if e == SyncError::Cancelled {
553            rep.outcome = PassOutcome::Cancelled;
554            return;
555        }
556        self.observer.event("pass_failed", &e.to_string());
557        let failures = self.store.ledger().consecutive_failures.saturating_add(1);
558        let secs = self
559            .config
560            .backoff
561            .delay_secs(failures, e.retry_hint_secs());
562        let until = self.clock.now_ms() + (secs * 1000.0) as i64;
563        let _ = self.store.mutate(|l| {
564            l.consecutive_failures = failures;
565            l.retry_after_ms = Some(until);
566        });
567        rep.retry_after_secs = Some(secs);
568        rep.outcome = match e {
569            SyncError::Unavailable(d) => PassOutcome::Unavailable(d),
570            other => PassOutcome::Failed(other.to_string()),
571        };
572    }
573
574    /// A conditional save lost a race. Read the server record, decide with
575    /// the conflict policy (tombstone always wins) and either re-save on top
576    /// of the server's change tag, take the remote, or keep both unchanged.
577    /// Bounded: a record that keeps changing under us stays pending.
578    fn resolve_conflict(
579        &mut self,
580        header: &ItemHeader,
581        item: &M::Item,
582        name: &str,
583        mut server: Option<RawRecord>,
584        generation: u64,
585        rep: &mut PassReport,
586    ) -> SyncResult<Resolved> {
587        for _ in 0..MAX_CONFLICT_ROUNDS {
588            self.check_current(generation)?;
589            let server_rec = match server.take() {
590                Some(r) => r,
591                None => match self.backend.fetch_record(name)? {
592                    Some(r) => r,
593                    None => return Ok(Resolved::Failed("conflicting record vanished".into())),
594                },
595            };
596            self.observer.event("save_conflict", name);
597            if let Some((id, deleted_at)) = read_tombstone(&server_rec) {
598                self.local.remove(&id)?;
599                self.store.mutate(|l| l.bury(&id, deleted_at))?;
600                rep.tombstones_applied += 1;
601                return Ok(Resolved::RemoteWon);
602            }
603            let remote_item = match self.mapper.decode(&server_rec) {
604                Ok(i) => i,
605                Err(e) => {
606                    rep.quarantined += 1;
607                    return Ok(Resolved::Failed(format!("server record undecodable: {e}")));
608                }
609            };
610            let rh = self.mapper.header(&remote_item);
611            if self
612                .policy
613                .upload_local_over_remote(header, item, &rh, &remote_item)
614            {
615                let mut rec = server_rec;
616                rec.set(field::ID, FieldValue::Text(header.id.clone()));
617                rec.set(field::CREATED_AT, FieldValue::Date(header.created_at_ms));
618                rec.set(field::UPDATED_AT, FieldValue::Date(header.updated_at_ms));
619                rec.fields.remove(field::DELETED_AT);
620                self.mapper.encode(item, &mut rec);
621                let outcomes = self
622                    .backend
623                    .save_records(&[rec], SavePolicy::IfServerUnchanged)?;
624                let Some(o) = outcomes.into_iter().next() else {
625                    return Ok(Resolved::Failed("no save outcome".into()));
626                };
627                match o.result {
628                    Ok(()) => return Ok(Resolved::Saved),
629                    Err(SyncError::Conflict) => server = o.server_record,
630                    Err(e) => return Ok(Resolved::Failed(e.to_string())),
631                }
632                continue;
633            }
634            let version = RemoteVersion {
635                record_name: name.to_string(),
636                updated_at_ms: rh.updated_at_ms,
637            };
638            if self.policy.accept_remote_over_local(header, &rh) {
639                self.check_current(generation)?;
640                match self.local.apply_remote(&[remote_item])?.into_iter().next() {
641                    Some(ApplyOutcome::Deferred(_)) => {
642                        let id = header.id.clone();
643                        self.store.mutate(|l| {
644                            l.pending_apply.insert(id, server_rec);
645                        })?;
646                    }
647                    _ => rep.downloaded += 1,
648                }
649            }
650            let id = header.id.clone();
651            self.store.mutate(|l| {
652                l.pending_uploads.remove(&id);
653                l.remote_versions.insert(id, version);
654            })?;
655            return Ok(Resolved::RemoteWon);
656        }
657        Ok(Resolved::Failed("conflict retries exhausted".into()))
658    }
659
660    /// Bind the ledger to the signed-in account scope; reset it when the
661    /// scope changed so one account's cursor, versions and queued deletions
662    /// are never applied to another.
663    ///
664    /// With `require_consent_on_scope_change` a switch from a bound scope is not applied: the
665    /// ledger is left untouched, the request is persisted and `self.gate` is set so the pass
666    /// stops before any I/O.
667    fn bind_account(&mut self, generation: u64) -> SyncResult<()> {
668        self.check_current(generation)?;
669        let Some(scope) = self.auth.account_scope() else {
670            return Ok(());
671        };
672        match self.store.ledger().scope.clone() {
673            Some(bound) if bound == scope => {
674                if self.store.ledger().pending_consent.is_some() {
675                    // The account went back to the bound one: the request is moot.
676                    self.store.mutate(|l| l.pending_consent = None)?;
677                }
678                Ok(())
679            }
680            Some(bound) if self.config.require_consent_on_scope_change => {
681                let req = ConsentRequest {
682                    from: bound,
683                    to: scope,
684                };
685                if self.store.ledger().pending_consent.as_ref() != Some(&req) {
686                    let stored = req.clone();
687                    self.store.mutate(|l| l.pending_consent = Some(stored))?;
688                    self.observer
689                        .event("consent_required", &format!("{} -> {}", req.from, req.to));
690                }
691                self.gate = Some(req);
692                Ok(())
693            }
694            None => {
695                // Ledger from before scoping: keep tombstones (they only
696                // suppress local copies); never import its cursor, versions,
697                // aliases or queued deletions into this account.
698                let dropped = {
699                    let l = self.store.ledger();
700                    l.change_token.is_some()
701                        || !l.remote_versions.is_empty()
702                        || !l.aliases.is_empty()
703                        || !l.pending_tombstones.is_empty()
704                };
705                self.store.mutate(|l| {
706                    l.scope = Some(scope);
707                    l.change_token = None;
708                    l.remote_versions.clear();
709                    l.aliases.clear();
710                    l.pending_tombstones.clear();
711                    l.has_synced = false;
712                })?;
713                if dropped {
714                    self.observer
715                        .event("legacy_ledger_unscoped", "pre-scope ledger state dropped");
716                }
717                Ok(())
718            }
719            Some(bound) => {
720                self.hooks.on_account_switch(&bound, &scope);
721                self.store.mutate(|l| {
722                    *l = crate::ledger::Ledger::default();
723                    l.scope = Some(scope.clone());
724                })?;
725                self.observer.event("account_switched", "ledger reset");
726                Ok(())
727            }
728        }
729    }
730
731    fn check_current(&self, generation: u64) -> SyncResult<()> {
732        if self.auth.enabled() && self.auth.generation() == generation {
733            Ok(())
734        } else {
735            Err(SyncError::Cancelled)
736        }
737    }
738
739    /// Returns `Some(reason)` when the pass finished with queued work left.
740    fn pass_inner(&mut self, generation: u64, rep: &mut PassReport) -> SyncResult<Option<String>> {
741        self.auth.availability()?;
742        // `availability` refreshes the account identity, which may bump the
743        // generation; the pass belongs to the generation observed after it.
744        let generation = self.auth.generation().max(generation);
745        self.bind_account(generation)?;
746        if self.gate.is_some() {
747            return Ok(None);
748        }
749        self.backend.prepare()?;
750        self.check_current(generation)?;
751
752        // Domain announce-style records go first; failure fails the pass.
753        let mut extras = self.store.ledger().extras.clone();
754        let outbound = self.hooks.outbound(&mut extras);
755        if !outbound.is_empty() {
756            self.check_current(generation)?;
757            self.write_required(&outbound)?;
758        }
759        self.check_current(generation)?;
760        self.store.mutate(|l| l.extras = extras)?;
761
762        // One-shot full replays (repair migrations) reset only the cursor.
763        if !self.replay_flags.is_empty() {
764            self.store.mutate(|l| l.change_token = None)?;
765        }
766
767        let fetched = self.fetch_all(generation)?;
768        let mapper_type = self.mapper.record_type().to_string();
769        let mut entries: Vec<RemoteEntry> = Vec::new();
770        let mut foreign: Vec<RawRecord> = Vec::new();
771        for record in fetched.records {
772            if record.record_type != mapper_type {
773                foreign.push(record);
774                continue;
775            }
776            if let Some((id, deleted_at)) = read_tombstone(&record) {
777                entries.push(RemoteEntry {
778                    id,
779                    record_name: record.record_name.clone(),
780                    version_ms: deleted_at,
781                    state: RemoteState::Tombstone,
782                    record,
783                });
784                continue;
785            }
786            match self.mapper.decode(&record) {
787                Ok(item) => {
788                    let h = self.mapper.header(&item);
789                    entries.push(RemoteEntry {
790                        id: h.id.clone(),
791                        record_name: record.record_name.clone(),
792                        version_ms: h.updated_at_ms,
793                        state: RemoteState::Live(h),
794                        record,
795                    });
796                }
797                Err(why) => {
798                    // A bad record never blocks the page (both platforms).
799                    rep.quarantined += 1;
800                    self.observer.event(
801                        "record_quarantined",
802                        &format!("{}: {why}", record.record_name),
803                    );
804                }
805            }
806        }
807        // Server-side hard deletions of records we know behave as tombstones.
808        for name in &fetched.deleted {
809            let known = self
810                .store
811                .ledger()
812                .aliases
813                .iter()
814                .find(|(_, names)| names.contains(name))
815                .map(|(id, _)| id.clone())
816                .or_else(|| {
817                    self.store
818                        .ledger()
819                        .remote_versions
820                        .contains_key(name)
821                        .then(|| name.clone())
822                });
823            if let Some(id) = known {
824                if !entries.iter().any(|e| e.id == id && e.is_tombstone()) {
825                    let mut record = RawRecord::new(&mapper_type, name);
826                    let now = self.clock.now_ms();
827                    write_tombstone(&mut record, &id, now, &[]);
828                    entries.push(RemoteEntry {
829                        id,
830                        record_name: name.clone(),
831                        version_ms: now,
832                        state: RemoteState::Tombstone,
833                        record,
834                    });
835                }
836            }
837        }
838        if rep.quarantined > 0 {
839            let n = rep.quarantined as u64;
840            self.store.mutate(|l| l.quarantined_total += n)?;
841        }
842
843        // Aliases: every record name ever seen for a logical id.
844        self.store.mutate(|l| {
845            for e in &entries {
846                l.aliases
847                    .entry(e.id.clone())
848                    .or_default()
849                    .insert(e.record_name.clone());
850            }
851        })?;
852
853        // Reduce duplicate records of one logical id.
854        let mut canonical: BTreeMap<String, RemoteEntry> = BTreeMap::new();
855        for entry in entries {
856            match canonical.get(&entry.id) {
857                Some(cur) if cur.rank() >= entry.rank() => {}
858                _ => {
859                    canonical.insert(entry.id.clone(), entry);
860                }
861            }
862        }
863
864        // Domain-owned record types (queue requests etc.).
865        if !foreign.is_empty() {
866            let mut extras = self.store.ledger().extras.clone();
867            let responses = self.hooks.on_foreign_records(&foreign, &mut extras);
868            // Responses are durable on the zone BEFORE the extras that record
869            // them are persisted, and a failed write aborts the pass before
870            // the change token can pass the request: the delta replays and
871            // the idempotent hook answers again. Never acknowledge a request
872            // whose response never landed.
873            if !responses.is_empty() {
874                self.check_current(generation)?;
875                if let Err(e) = self.write_required(&responses) {
876                    self.observer.event("foreign_write_failed", &e.to_string());
877                    return Err(e);
878                }
879            }
880            self.check_current(generation)?;
881            self.store.mutate(|l| l.extras = extras)?;
882        }
883
884        // Remote tombstones are authoritative: delete locally, block id.
885        let tombs: Vec<(String, i64)> = canonical
886            .values()
887            .filter(|e| e.is_tombstone())
888            .map(|e| (e.id.clone(), e.version_ms))
889            .collect();
890        for (id, at) in &tombs {
891            self.check_current(generation)?;
892            self.local.remove(id)?;
893            let (id, at) = (id.clone(), *at);
894            self.store.mutate(|l| {
895                l.bury(&id, at);
896                l.pending_tombstones.remove(&id);
897            })?;
898        }
899        rep.tombstones_applied = tombs.len();
900        self.check_current(generation)?;
901
902        let mut pending: Vec<String> = Vec::new();
903
904        // Our own queued deletions.
905        match self.flush_tombstones(&canonical, generation, rep) {
906            Ok(true) => {}
907            Ok(false) => pending.push("tombstones pending".into()),
908            Err(e) => return Err(e),
909        }
910
911        // ---- Local scan ----
912        // One paged pass over the local store feeds the inferred deletions, the stale-queue
913        // check, the upload plan and the download comparison, so at most one page of local
914        // items is resident; only items that must upload are kept, and only the headers of
915        // items that also have a live remote entry.
916        struct Upload<I> {
917            header: ItemHeader,
918            item: I,
919            base: Option<RawRecord>,
920            is_new: bool,
921        }
922        let infer = self.config.infer_local_deletions;
923        let mut gone_candidates: HashSet<String> = if infer {
924            let l = self.store.ledger();
925            l.remote_versions
926                .keys()
927                .filter(|id| !l.tombstones.contains_key(*id))
928                .filter(|id| !l.pending_apply.contains_key(*id))
929                .cloned()
930                .collect()
931        } else {
932            HashSet::new()
933        };
934        let mut queued_unseen: HashSet<String> = self
935            .store
936            .ledger()
937            .pending_uploads
938            .iter()
939            .cloned()
940            .collect();
941        let mut matched: HashMap<String, ItemHeader> = HashMap::new();
942        let mut uploads: Vec<Upload<M::Item>> = Vec::new();
943        let mut failed_uploads = 0usize;
944        self.scan_local(|item| {
945            let header = self.mapper.header(&item);
946            if infer {
947                gone_candidates.remove(&header.id);
948            }
949            queued_unseen.remove(&header.id);
950            let canon = canonical.get(&header.id);
951            if let Some(RemoteEntry {
952                state: RemoteState::Live(_),
953                ..
954            }) = canon
955            {
956                matched
957                    .entry(header.id.clone())
958                    .or_insert_with(|| header.clone());
959            }
960            if self.store.ledger().tombstones.contains_key(&header.id) {
961                rep.blocked_reuploads += 1;
962                return Ok(());
963            }
964            if self.store.ledger().withheld.contains(&header.id) {
965                return Ok(());
966            }
967            let queued = self.store.ledger().pending_uploads.contains(&header.id);
968            match canon {
969                Some(RemoteEntry {
970                    state: RemoteState::Live(rh),
971                    record,
972                    ..
973                }) => {
974                    if let Ok(remote_item) = self.mapper.decode(record) {
975                        if self
976                            .policy
977                            .upload_local_over_remote(&header, &item, rh, &remote_item)
978                        {
979                            uploads.push(Upload {
980                                header,
981                                item,
982                                base: Some(record.clone()),
983                                is_new: false,
984                            });
985                        }
986                    }
987                }
988                Some(_) => {}
989                None => {
990                    if let Some(known) =
991                        self.store.ledger().remote_versions.get(&header.id).cloned()
992                    {
993                        // Cursor delta did not include it: unchanged remotely.
994                        if known.updated_at_ms < header.updated_at_ms || queued {
995                            match self.backend.fetch_record(&known.record_name) {
996                                Ok(base) => uploads.push(Upload {
997                                    header,
998                                    item,
999                                    base,
1000                                    is_new: false,
1001                                }),
1002                                Err(e) => {
1003                                    failed_uploads += 1;
1004                                    self.observer.event("record_fetch_failed", &e.to_string());
1005                                }
1006                            }
1007                        }
1008                    } else {
1009                        uploads.push(Upload {
1010                            header,
1011                            item,
1012                            base: None,
1013                            is_new: true,
1014                        });
1015                    }
1016                }
1017            }
1018            Ok(())
1019        })?;
1020
1021        // Inferred deletions (desktop): published item vanished locally.
1022        if infer {
1023            let gone: Vec<String> = gone_candidates.into_iter().collect();
1024            if !gone.is_empty() {
1025                let now = self.clock.now_ms();
1026                self.store.mutate(|l| {
1027                    for id in gone {
1028                        l.bury(&id, now);
1029                        l.pending_tombstones.insert(id);
1030                    }
1031                })?;
1032                if !self.flush_tombstones(&canonical, generation, rep)? {
1033                    pending.push("tombstones pending".into());
1034                }
1035            }
1036        }
1037
1038        // ---- Uploads ----
1039        // Drop queued ids whose item no longer exists locally.
1040        let stale: Vec<String> = queued_unseen
1041            .into_iter()
1042            .filter(|id| self.store.ledger().pending_uploads.contains(id))
1043            .collect();
1044        if !stale.is_empty() {
1045            self.store.mutate(|l| {
1046                for id in stale {
1047                    l.pending_uploads.remove(&id);
1048                }
1049            })?;
1050        }
1051        // Oldest origin first, ties by record name: deterministic history.
1052        uploads.sort_by(|a, b| {
1053            (
1054                a.header.created_at_ms,
1055                self.mapper.record_name(&a.header.id),
1056            )
1057                .cmp(&(
1058                    b.header.created_at_ms,
1059                    self.mapper.record_name(&b.header.id),
1060                ))
1061        });
1062        let (new_ones, existing): (Vec<_>, Vec<_>) = uploads.into_iter().partition(|u| u.is_new);
1063        let size = self.config.batch_size.min(self.backend.max_batch()).max(1);
1064        let mut chunks: Vec<(bool, Vec<Upload<M::Item>>)> = Vec::new();
1065        if self.config.ordered_new_publication {
1066            chunks.extend(new_ones.into_iter().map(|u| (true, vec![u])));
1067            let mut it = existing.into_iter().peekable();
1068            while it.peek().is_some() {
1069                chunks.push((false, it.by_ref().take(size).collect()));
1070            }
1071        } else {
1072            let mut it = new_ones.into_iter().chain(existing).peekable();
1073            while it.peek().is_some() {
1074                chunks.push((false, it.by_ref().take(size).collect()));
1075            }
1076        }
1077
1078        let mut blocked_new = false;
1079        for (ordered_new, chunk) in chunks {
1080            if ordered_new && blocked_new {
1081                continue;
1082            }
1083            self.check_current(generation)?;
1084            let mut records = Vec::with_capacity(chunk.len());
1085            let mut by_name: BTreeMap<String, (String, i64)> = BTreeMap::new();
1086            for u in &chunk {
1087                let name = self.mapper.record_name(&u.header.id);
1088                let mut rec = u
1089                    .base
1090                    .clone()
1091                    .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name));
1092                rec.set(field::ID, FieldValue::Text(u.header.id.clone()));
1093                rec.set(field::CREATED_AT, FieldValue::Date(u.header.created_at_ms));
1094                rec.set(field::UPDATED_AT, FieldValue::Date(u.header.updated_at_ms));
1095                rec.fields.remove(field::DELETED_AT);
1096                self.mapper.encode(&u.item, &mut rec);
1097                by_name.insert(
1098                    rec.record_name.clone(),
1099                    (u.header.id.clone(), u.header.updated_at_ms),
1100                );
1101                records.push(rec);
1102            }
1103            // Conditional: an edit based on a stale change tag (or an insert of
1104            // a name that now exists) is refused, never overwritten.
1105            let outcomes = self
1106                .backend
1107                .save_records(&records, SavePolicy::IfServerUnchanged)?;
1108            let mut saved: Vec<(String, String, i64)> = Vec::new();
1109            let mut chunk_failed = false;
1110            for o in outcomes {
1111                let Some((id, version)) = by_name.get(&o.record_name) else {
1112                    continue;
1113                };
1114                match o.result {
1115                    Ok(()) => saved.push((id.clone(), o.record_name.clone(), *version)),
1116                    Err(SyncError::Conflict) => {
1117                        let upload = chunk
1118                            .iter()
1119                            .find(|u| self.mapper.record_name(&u.header.id) == o.record_name)
1120                            .expect("outcome names an upload of this chunk");
1121                        let (header, item) = (upload.header.clone(), upload.item.clone());
1122                        match self.resolve_conflict(
1123                            &header,
1124                            &item,
1125                            &o.record_name,
1126                            o.server_record,
1127                            generation,
1128                            rep,
1129                        )? {
1130                            Resolved::Saved => {
1131                                saved.push((id.clone(), o.record_name.clone(), *version))
1132                            }
1133                            Resolved::RemoteWon => {}
1134                            Resolved::Failed(why) => {
1135                                chunk_failed = true;
1136                                failed_uploads += 1;
1137                                self.observer.event(
1138                                    "record_save_failed",
1139                                    &format!("{}: {why}", o.record_name),
1140                                );
1141                            }
1142                        }
1143                    }
1144                    Err(e) => {
1145                        chunk_failed = true;
1146                        failed_uploads += 1;
1147                        self.observer
1148                            .event("record_save_failed", &format!("{}: {e}", o.record_name));
1149                    }
1150                }
1151            }
1152            if ordered_new && chunk_failed {
1153                blocked_new = true;
1154            }
1155            // Persist progress per chunk: a crash resumes from here.
1156            rep.uploaded += saved.len();
1157            self.store.mutate(|l| {
1158                for (id, name, version) in saved {
1159                    l.pending_uploads.remove(&id);
1160                    l.aliases
1161                        .entry(id.clone())
1162                        .or_default()
1163                        .insert(name.clone());
1164                    l.remote_versions.insert(
1165                        id,
1166                        RemoteVersion {
1167                            record_name: name,
1168                            updated_at_ms: version,
1169                        },
1170                    );
1171                }
1172            })?;
1173        }
1174        if failed_uploads > 0 {
1175            pending.push(format!("{failed_uploads} upload(s) pending"));
1176        }
1177
1178        // ---- Downloads ----
1179        // Plan first with ids and sort keys only (the global (created, id) order is kept), then
1180        // decode and apply a bounded chunk at a time so item bodies are never all resident.
1181        enum Source {
1182            Remote,
1183            Stash(RawRecord),
1184        }
1185        struct Planned {
1186            created: i64,
1187            id: String,
1188            source: Source,
1189        }
1190        let stash_ids: HashSet<String> =
1191            self.store.ledger().pending_apply.keys().cloned().collect();
1192        let mut taken_stash_ids: HashSet<String> = HashSet::new();
1193        let mut plan: Vec<Planned> = Vec::new();
1194        for entry in canonical.values() {
1195            let RemoteState::Live(rh) = &entry.state else {
1196                continue;
1197            };
1198            if self.store.ledger().tombstones.contains_key(&entry.id) {
1199                continue;
1200            }
1201            let take = match matched.get(&entry.id) {
1202                None => true,
1203                Some(lh) => self.policy.accept_remote_over_local(lh, rh),
1204            };
1205            if take {
1206                if stash_ids.contains(&entry.id) {
1207                    taken_stash_ids.insert(entry.id.clone());
1208                }
1209                plan.push(Planned {
1210                    created: rh.created_at_ms,
1211                    id: entry.id.clone(),
1212                    source: Source::Remote,
1213                });
1214            }
1215        }
1216        drop(matched);
1217        // Retry records whose local apply was deferred earlier.
1218        let stashed: Vec<(String, RawRecord)> = self
1219            .store
1220            .ledger()
1221            .pending_apply
1222            .iter()
1223            .map(|(k, v)| (k.clone(), v.clone()))
1224            .collect();
1225        for (id, record) in stashed {
1226            if taken_stash_ids.contains(&id) || self.store.ledger().tombstones.contains_key(&id) {
1227                continue;
1228            }
1229            if let Ok(item) = self.mapper.decode(&record) {
1230                let h = self.mapper.header(&item);
1231                plan.push(Planned {
1232                    created: h.created_at_ms,
1233                    id: h.id,
1234                    source: Source::Stash(record),
1235                });
1236            }
1237        }
1238        plan.sort_by(|a, b| (a.created, a.id.as_str()).cmp(&(b.created, b.id.as_str())));
1239        let mut deferred = 0usize;
1240        for chunk in plan.chunks(APPLY_CHUNK) {
1241            self.check_current(generation)?;
1242            let mut items: Vec<M::Item> = Vec::with_capacity(chunk.len());
1243            let mut decoded_at: Vec<usize> = Vec::with_capacity(chunk.len());
1244            for (i, p) in chunk.iter().enumerate() {
1245                let decoded = match &p.source {
1246                    Source::Remote => canonical.get(&p.id).map(|e| self.mapper.decode(&e.record)),
1247                    Source::Stash(rec) => Some(self.mapper.decode(rec)),
1248                };
1249                if let Some(Ok(item)) = decoded {
1250                    items.push(item);
1251                    decoded_at.push(i);
1252                }
1253            }
1254            if items.is_empty() {
1255                continue;
1256            }
1257            let outcomes = self.local.apply_remote(&items)?;
1258            if outcomes.len() != items.len() {
1259                return Err(SyncError::Persistence(
1260                    "apply_remote returned wrong outcome count".into(),
1261                ));
1262            }
1263            drop(items);
1264            let mut applied_ids: Vec<String> = Vec::new();
1265            let mut stash: Vec<(String, RawRecord)> = Vec::new();
1266            for (i, out) in decoded_at.into_iter().zip(outcomes) {
1267                let p = &chunk[i];
1268                match out {
1269                    ApplyOutcome::Applied => {
1270                        rep.downloaded += 1;
1271                        applied_ids.push(p.id.clone());
1272                    }
1273                    ApplyOutcome::Deferred(why) => {
1274                        deferred += 1;
1275                        self.observer
1276                            .event("apply_deferred", &format!("{}: {why}", p.id));
1277                        let record = match &p.source {
1278                            Source::Remote => canonical.get(&p.id).map(|e| e.record.clone()),
1279                            Source::Stash(rec) => Some(rec.clone()),
1280                        };
1281                        if let Some(record) = record {
1282                            stash.push((p.id.clone(), record));
1283                        }
1284                    }
1285                }
1286            }
1287            let touches_ledger = !stash.is_empty()
1288                || applied_ids
1289                    .iter()
1290                    .any(|id| self.store.ledger().pending_apply.contains_key(id));
1291            if touches_ledger {
1292                self.store.mutate(|l| {
1293                    for id in applied_ids {
1294                        l.pending_apply.remove(&id);
1295                    }
1296                    for (id, rec) in stash {
1297                        l.pending_apply.insert(id, rec);
1298                    }
1299                })?;
1300            }
1301        }
1302        drop(plan);
1303        if deferred > 0 {
1304            pending.push(format!("{deferred} download(s) deferred"));
1305        }
1306
1307        // Remote versions + advance the token: only now, after everything
1308        // above applied (or was durably queued).
1309        // Final guard: a pass whose account/enable generation changed during
1310        // I/O must not advance the cursor or mark the ledger synced.
1311        self.check_current(generation)?;
1312        let flags = std::mem::take(&mut self.replay_flags);
1313        let token = fetched.token;
1314        self.store.mutate(|l| {
1315            for e in canonical.values() {
1316                // A local upload this pass may already hold a newer version.
1317                let keep = l
1318                    .remote_versions
1319                    .get(&e.id)
1320                    .is_some_and(|k| k.updated_at_ms > e.version_ms);
1321                if !keep {
1322                    l.remote_versions.insert(
1323                        e.id.clone(),
1324                        RemoteVersion {
1325                            record_name: e.record_name.clone(),
1326                            updated_at_ms: e.version_ms,
1327                        },
1328                    );
1329                }
1330            }
1331            if token.is_some() {
1332                l.change_token = token;
1333            }
1334            l.has_synced = true;
1335            for f in flags {
1336                l.flags.insert(f);
1337            }
1338        })?;
1339        if rep.blocked_reuploads > 0 {
1340            self.observer
1341                .event("blocked_reupload", &rep.blocked_reuploads.to_string());
1342        }
1343        Ok(if pending.is_empty() {
1344            None
1345        } else {
1346            Some(pending.join("; "))
1347        })
1348    }
1349
1350    /// Fetch every page; reset the token once on expiry and re-prepare the
1351    /// zone once if it vanished.
1352    fn fetch_all(&mut self, generation: u64) -> SyncResult<Fetched> {
1353        let mut reset_token = false;
1354        let mut recreated = false;
1355        'restart: loop {
1356            let mut token = self.store.ledger().change_token.clone();
1357            let mut out = Fetched {
1358                records: Vec::new(),
1359                deleted: Vec::new(),
1360                token: None,
1361            };
1362            loop {
1363                self.check_current(generation)?;
1364                match self.backend.fetch_changes(token.as_deref()) {
1365                    Ok(page) => {
1366                        out.records.extend(page.records);
1367                        out.deleted.extend(page.deleted_record_names);
1368                        if page.new_token.is_some() {
1369                            token = page.new_token.clone();
1370                            out.token = page.new_token;
1371                        }
1372                        if !page.more_coming {
1373                            return Ok(out);
1374                        }
1375                    }
1376                    Err(SyncError::TokenExpired) if !reset_token && token.is_some() => {
1377                        reset_token = true;
1378                        self.observer
1379                            .event("token_reset", "change token expired; full replay");
1380                        self.store.mutate(|l| l.change_token = None)?;
1381                        continue 'restart;
1382                    }
1383                    Err(SyncError::ZoneMissing) if !recreated => {
1384                        recreated = true;
1385                        self.observer
1386                            .event("zone_recreated", "zone missing; re-preparing");
1387                        // Every remote record is gone: what the ledger believes is
1388                        // published no longer exists, so local items must
1389                        // re-publish (donor: acknowledgements are invalidated).
1390                        self.store.mutate(|l| {
1391                            l.change_token = None;
1392                            l.remote_versions.clear();
1393                            l.aliases.clear();
1394                        })?;
1395                        self.backend.prepare()?;
1396                        continue 'restart;
1397                    }
1398                    Err(e) => return Err(e),
1399                }
1400            }
1401        }
1402    }
1403
1404    /// Write queued tombstones to every alias record. Returns true when the
1405    /// queue drained.
1406    fn flush_tombstones(
1407        &mut self,
1408        canonical: &BTreeMap<String, RemoteEntry>,
1409        generation: u64,
1410        rep: &mut PassReport,
1411    ) -> SyncResult<bool> {
1412        let ids: Vec<String> = self
1413            .store
1414            .ledger()
1415            .pending_tombstones
1416            .iter()
1417            .cloned()
1418            .collect();
1419        if ids.is_empty() {
1420            return Ok(true);
1421        }
1422        struct Write {
1423            id: String,
1424            record: RawRecord,
1425        }
1426        let mut writes: Vec<Write> = Vec::new();
1427        let mut never_remote: Vec<String> = Vec::new();
1428        for id in &ids {
1429            let mut names: BTreeSet<String> = self
1430                .store
1431                .ledger()
1432                .aliases
1433                .get(id)
1434                .cloned()
1435                .unwrap_or_default();
1436            let known =
1437                self.store.ledger().remote_versions.contains_key(id) || canonical.contains_key(id);
1438            if names.is_empty() && !known {
1439                never_remote.push(id.clone());
1440                continue;
1441            }
1442            names.insert(self.mapper.record_name(id));
1443            let at = self
1444                .store
1445                .ledger()
1446                .tombstones
1447                .get(id)
1448                .copied()
1449                .unwrap_or_else(|| self.clock.now_ms());
1450            for name in names {
1451                let mut record = match canonical.get(id).filter(|e| e.record_name == name) {
1452                    Some(e) => e.record.clone(),
1453                    None => self
1454                        .backend
1455                        .fetch_record(&name)
1456                        .ok()
1457                        .flatten()
1458                        .unwrap_or_else(|| RawRecord::new(self.mapper.record_type(), &name)),
1459                };
1460                write_tombstone(&mut record, id, at, self.mapper.tombstone_keep_fields());
1461                writes.push(Write {
1462                    id: id.clone(),
1463                    record,
1464                });
1465            }
1466        }
1467        if !never_remote.is_empty() {
1468            // Created and deleted before ever reaching the zone: nothing to write.
1469            self.store.mutate(|l| {
1470                for id in never_remote {
1471                    l.pending_tombstones.remove(&id);
1472                }
1473            })?;
1474        }
1475        let mut failed: BTreeSet<String> = BTreeSet::new();
1476        let size = self
1477            .config
1478            .tombstone_batch
1479            .min(self.backend.max_batch())
1480            .max(1);
1481        for chunk in writes.chunks(size) {
1482            self.check_current(generation)?;
1483            let records: Vec<RawRecord> = chunk.iter().map(|w| w.record.clone()).collect();
1484            let outcomes = self.backend.save_records(&records, SavePolicy::Overwrite)?;
1485            for w in chunk {
1486                let ok = outcomes
1487                    .iter()
1488                    .find(|o| o.record_name == w.record.record_name)
1489                    .is_some_and(|o| o.result.is_ok());
1490                if !ok {
1491                    failed.insert(w.id.clone());
1492                    self.observer
1493                        .event("tombstone_save_failed", &w.record.record_name);
1494                }
1495            }
1496        }
1497        let acked: Vec<String> = writes
1498            .iter()
1499            .map(|w| w.id.clone())
1500            .collect::<BTreeSet<_>>()
1501            .into_iter()
1502            .filter(|id| !failed.contains(id))
1503            .collect();
1504        rep.tombstones_written += acked.len();
1505        self.store.mutate(|l| {
1506            for id in &acked {
1507                l.pending_tombstones.remove(id);
1508            }
1509        })?;
1510        Ok(failed.is_empty())
1511    }
1512
1513    /// Save records whose failure must fail the pass.
1514    fn write_required(&self, records: &[RawRecord]) -> SyncResult<()> {
1515        for chunk in records.chunks(self.backend.max_batch().max(1)) {
1516            for o in self.backend.save_records(chunk, SavePolicy::Overwrite)? {
1517                o.result?;
1518            }
1519        }
1520        Ok(())
1521    }
1522}